populateEmojisのリファクタと絵文字情報のキャッシュ (#7378)

* revert

* Refactor populateEmojis, Cache emojis

* ん

* fix typo

* コメント

src/misc/cache.ts

@@ -14,13 +14,30 @@ export class Cache<T> {
 		});
 	}
 
-	public get(key: string | null): T | null {
+	public get(key: string | null): T | undefined {
 		const cached = this.cache.get(key);
-		if (cached == null) return null;
+		if (cached == null) return undefined;
 		if ((Date.now() - cached.date) > this.lifetime) {
 			this.cache.delete(key);
-			return null;
+			return undefined;
 		}
 		return cached.value;
 	}
+
+	public delete(key: string | null) {
+		this.cache.delete(key);
+	}
+
+	public async fetch(key: string | null, fetcher: () => Promise<T>): Promise<T> {
+		const cachedValue = this.get(key);
+		if (cachedValue !== undefined) {
+			// Cache HIT
+			return cachedValue;
+		}
+
+		// Cache MISS
+		const value = await fetcher();
+		this.set(key, value);
+		return value;
+	}
 }

src/misc/populate-emojis.ts

@@ -0,0 +1,58 @@
+import { Emojis } from '../models';
+import { Emoji } from '../models/entities/emoji';
+import { Cache } from './cache';
+import { isSelfHost, toPunyNullable } from './convert-host';
+
+const cache = new Cache<Emoji | null>(1000 * 60 * 60);
+
+/**
+ * 添付用絵文字情報
+ */
+type PopulatedEmoji = {
+	name: string;
+	url: string;
+};
+
+/**
+ * 添付用絵文字情報を解決する
+ * @param emojiName ノートやユーザープロフィールに添付された、またはリアクションのカスタム絵文字名 (:は含めない, リアクションでローカルホストの場合は@.を付ける (これはdecodeReactionで可能))
+ * @param noteUserHost ノートやユーザープロフィールの所有者
+ * @returns 絵文字情報, nullは未マッチを意味する
+ */
+export async function populateEmoji(emojiName: string, noteUserHost: string | null): Promise<PopulatedEmoji | null> {
+	const match = emojiName.match(/^(\w+)(?:@([\w.-]+))?$/);
+	if (!match) return null;
+
+	const name = match[1];
+
+	// クエリに使うホスト
+	let host = match[2] === '.' ? null	// .はローカルホスト (ここがマッチするのはリアクションのみ)
+		: match[2] === undefined ? noteUserHost	// ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない)
+		: isSelfHost(match[2]) ? null	// 自ホスト指定
+		: (match[2] || noteUserHost);	// 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない)
+
+	host = toPunyNullable(host);
+
+	const queryOrNull = async () => (await Emojis.findOne({
+		name,
+		host
+	})) || null;
+
+	const emoji = await cache.fetch(`${name} ${host}`, queryOrNull);
+
+	if (emoji == null) return null;
+
+	return {
+		name: emojiName,
+		url: emoji.url,
+	};
+}
+
+/**
+ * 複数の添付用絵文字情報を解決する (キャシュ付き, 存在しないものは結果から除外される)
+ */
+export async function populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<PopulatedEmoji[]> {
+	const emojis = await Promise.all(emojiNames.map(x => populateEmoji(x, noteUserHost)));
+	return emojis.filter((x): x is PopulatedEmoji => x != null);
+}
+

src/models/repositories/note.ts

@@ -1,15 +1,14 @@
 import { EntityRepository, Repository, In } from 'typeorm';
 import { Note } from '../entities/note';
 import { User } from '../entities/user';
-import { Emojis, Users, PollVotes, DriveFiles, NoteReactions, Followings, Polls, Channels } from '..';
+import { Users, PollVotes, DriveFiles, NoteReactions, Followings, Polls, Channels } from '..';
 import { SchemaType } from '../../misc/schema';
 import { awaitAll } from '../../prelude/await-all';
 import { convertLegacyReaction, convertLegacyReactions, decodeReaction } from '../../misc/reaction-lib';
 import { toString } from '../../mfm/to-string';
 import { parse } from '../../mfm/parse';
-import { Emoji } from '../entities/emoji';
-import { concat } from '../../prelude/array';
 import { NoteReaction } from '../entities/note-reaction';
+import { populateEmojis } from '../../misc/populate-emojis';
 
 export type PackedNote = SchemaType<typeof packedNoteSchema>;
 
@@ -85,7 +84,6 @@ export class NoteRepository extends Repository<Note> {
 			detail?: boolean;
 			skipHide?: boolean;
 			_hint_?: {
-				emojis: Emoji[] | null;
 				myReactions: Map<Note['id'], NoteReaction | null>;
 			};
 		}
@@ -135,93 +133,6 @@ export class NoteRepository extends Repository<Note> {
 			};
 		}
 
-		/**
-		 * 添付用emojisを解決する
-		 * @param emojiNames Note等に添付されたカスタム絵文字名 (:は含めない)
-		 * @param noteUserHost Noteのホスト
-		 * @param reactionNames Note等にリアクションされたカスタム絵文字名 (:は含めない)
-		 */
-		async function populateEmojis(emojiNames: string[], noteUserHost: string | null, reactionNames: string[]) {
-			const customReactions = reactionNames?.map(x => decodeReaction(x)).filter(x => x.name);
-
-			let all = [] as {
-				name: string,
-				url: string
-			}[];
-
-			// 与えられたhintだけで十分(=新たにクエリする必要がない)かどうかを表すフラグ
-			let enough = true;
-			if (options?._hint_?.emojis) {
-				for (const name of emojiNames) {
-					const matched = options._hint_.emojis.find(x => x.name === name && x.host === noteUserHost);
-					if (matched) {
-						all.push({
-							name: matched.name,
-							url: matched.url,
-						});
-					} else {
-						enough = false;
-					}
-				}
-				for (const customReaction of customReactions) {
-					const matched = options._hint_.emojis.find(x => x.name === customReaction.name && x.host === customReaction.host);
-					if (matched) {
-						all.push({
-							name: `${matched.name}@${matched.host || '.'}`,	// @host付きでローカルは.
-							url: matched.url,
-						});
-					} else {
-						enough = false;
-					}
-				}
-			} else {
-				enough = false;
-			}
-			if (enough) return all;
-
-			// カスタム絵文字
-			if (emojiNames?.length > 0) {
-				const tmp = await Emojis.find({
-					where: {
-						name: In(emojiNames),
-						host: noteUserHost
-					},
-					select: ['name', 'host', 'url']
-				}).then(emojis => emojis.map((emoji: Emoji) => {
-					return {
-						name: emoji.name,
-						url: emoji.url,
-					};
-				}));
-
-				all = concat([all, tmp]);
-			}
-
-			if (customReactions?.length > 0) {
-				const where = [] as {}[];
-
-				for (const customReaction of customReactions) {
-					where.push({
-						name: customReaction.name,
-						host: customReaction.host
-					});
-				}
-
-				const tmp = await Emojis.find({
-					where,
-					select: ['name', 'host', 'url']
-				}).then(emojis => emojis.map((emoji: Emoji) => {
-					return {
-						name: `${emoji.name}@${emoji.host || '.'}`,	// @host付きでローカルは.
-						url: emoji.url,
-					};
-				}));
-				all = concat([all, tmp]);
-			}
-
-			return all;
-		}
-
 		async function populateMyReaction() {
 			if (options?._hint_?.myReactions) {
 				const reaction = options._hint_.myReactions.get(note.id);
@@ -257,15 +168,14 @@ export class NoteRepository extends Repository<Note> {
 				: await Channels.findOne(note.channelId)
 			: null;
 
+		const reactionEmojiNames = Object.keys(note.reactions).filter(x => x?.startsWith(':')).map(x => decodeReaction(x).reaction).map(x => x.replace(/:/g, ''));
+
 		const packed = await awaitAll({
 			id: note.id,
 			createdAt: note.createdAt.toISOString(),
 			userId: note.userId,
 			user: Users.pack(note.user || note.userId, meId, {
 				detail: false,
-				_hint_: {
-					emojis: options?._hint_?.emojis || null
-				}
 			}),
 			text: text,
 			cw: note.cw,
@@ -277,7 +187,7 @@ export class NoteRepository extends Repository<Note> {
 			repliesCount: note.repliesCount,
 			reactions: convertLegacyReactions(note.reactions),
 			tags: note.tags.length > 0 ? note.tags : undefined,
-			emojis: populateEmojis(note.emojis, host, Object.keys(note.reactions)),
+			emojis: populateEmojis(note.emojis.concat(reactionEmojiNames), host),
 			fileIds: note.fileIds,
 			files: DriveFiles.packMany(note.fileIds),
 			replyId: note.replyId,
@@ -350,48 +260,10 @@ export class NoteRepository extends Repository<Note> {
 			}
 		}
 
-		// TODO: ここら辺の処理をaggregateEmojisみたいな関数に切り出したい
-		let emojisWhere: any[] = [];
-		for (const note of notes) {
-			if (typeof note !== 'object') continue;
-			emojisWhere.push({
-				name: In(note.emojis),
-				host: note.userHost
-			});
-			if (note.renote) {
-				emojisWhere.push({
-					name: In(note.renote.emojis),
-					host: note.renote.userHost
-				});
-				if (note.renote.user) {
-					emojisWhere.push({
-						name: In(note.renote.user.emojis),
-						host: note.renote.userHost
-					});
-				}
-			}
-			const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name);
-			emojisWhere = emojisWhere.concat(customReactions.map(x => ({
-				name: x.name,
-				host: x.host
-			})));
-			if (note.user) {
-				emojisWhere.push({
-					name: In(note.user.emojis),
-					host: note.userHost
-				});
-			}
-		}
-		const emojis = emojisWhere.length > 0 ? await Emojis.find({
-			where: emojisWhere,
-			select: ['name', 'host', 'url']
-		}) : null;
-
 		return await Promise.all(notes.map(n => this.pack(n, me, {
 			...options,
 			_hint_: {
-				myReactions: myReactionsMap,
+				myReactions: myReactionsMap
-				emojis: emojis
 			}
 		})));
 	}

src/models/repositories/notification.ts

@@ -1,13 +1,11 @@
 import { EntityRepository, In, Repository } from 'typeorm';
-import { Users, Notes, UserGroupInvitations, AccessTokens, NoteReactions, Emojis } from '..';
+import { Users, Notes, UserGroupInvitations, AccessTokens, NoteReactions } from '..';
 import { Notification } from '../entities/notification';
 import { awaitAll } from '../../prelude/await-all';
 import { SchemaType } from '../../misc/schema';
 import { Note } from '../entities/note';
 import { NoteReaction } from '../entities/note-reaction';
 import { User } from '../entities/user';
-import { decodeReaction } from '../../misc/reaction-lib';
-import { Emoji } from '../entities/emoji';
 
 export type PackedNotification = SchemaType<typeof packedNotificationSchema>;
 
@@ -17,7 +15,6 @@ export class NotificationRepository extends Repository<Notification> {
 		src: Notification['id'] | Notification,
 		options: {
 			_hintForEachNotes_?: {
-				emojis: Emoji[] | null;
 				myReactions: Map<Note['id'], NoteReaction | null>;
 			};
 		}
@@ -101,47 +98,9 @@ export class NotificationRepository extends Repository<Notification> {
 			myReactionsMap.set(target, myReactions.find(reaction => reaction.noteId === target) || null);
 		}
 
-		// TODO: ここら辺の処理をaggregateEmojisみたいな関数に切り出したい
-		let emojisWhere: any[] = [];
-		for (const note of notes) {
-			if (typeof note !== 'object') continue;
-			emojisWhere.push({
-				name: In(note.emojis),
-				host: note.userHost
-			});
-			if (note.renote) {
-				emojisWhere.push({
-					name: In(note.renote.emojis),
-					host: note.renote.userHost
-				});
-				if (note.renote.user) {
-					emojisWhere.push({
-						name: In(note.renote.user.emojis),
-						host: note.renote.userHost
-					});
-				}
-			}
-			const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name);
-			emojisWhere = emojisWhere.concat(customReactions.map(x => ({
-				name: x.name,
-				host: x.host
-			})));
-			if (note.user) {
-				emojisWhere.push({
-					name: In(note.user.emojis),
-					host: note.userHost
-				});
-			}
-		}
-		const emojis = emojisWhere.length > 0 ? await Emojis.find({
-			where: emojisWhere,
-			select: ['name', 'host', 'url']
-		}) : null;
-
 		return await Promise.all(notifications.map(x => this.pack(x, {
 			_hintForEachNotes_: {
-				myReactions: myReactionsMap,
+				myReactions: myReactionsMap
-				emojis: emojis,
 			}
 		})));
 	}

src/models/repositories/user.ts

@@ -1,11 +1,11 @@
 import $ from 'cafy';
 import { EntityRepository, Repository, In, Not } from 'typeorm';
 import { User, ILocalUser, IRemoteUser } from '../entities/user';
-import { Emojis, Notes, NoteUnreads, FollowRequests, Notifications, MessagingMessages, UserNotePinings, Followings, Blockings, Mutings, UserProfiles, UserSecurityKeys, UserGroupJoinings, Pages, Announcements, AnnouncementReads, Antennas, AntennaNotes, ChannelFollowings, Instances } from '..';
+import { Notes, NoteUnreads, FollowRequests, Notifications, MessagingMessages, UserNotePinings, Followings, Blockings, Mutings, UserProfiles, UserSecurityKeys, UserGroupJoinings, Pages, Announcements, AnnouncementReads, Antennas, AntennaNotes, ChannelFollowings, Instances } from '..';
 import config from '../../config';
 import { SchemaType } from '../../misc/schema';
 import { awaitAll } from '../../prelude/await-all';
-import { Emoji } from '../entities/emoji';
+import { populateEmojis } from '../../misc/populate-emojis';
 
 export type PackedUser = SchemaType<typeof packedUserSchema>;
 
@@ -150,9 +150,6 @@ export class UserRepository extends Repository<User> {
 		options?: {
 			detail?: boolean,
 			includeSecrets?: boolean,
-			_hint_?: {
-				emojis: Emoji[] | null;
-			};
 		}
 	): Promise<PackedUser> {
 		const opts = Object.assign({
@@ -170,34 +167,6 @@ export class UserRepository extends Repository<User> {
 		}) : [];
 		const profile = opts.detail ? await UserProfiles.findOneOrFail(user.id) : null;
 
-		let emojis: Emoji[] = [];
-		if (user.emojis.length > 0) {
-			// 与えられたhintだけで十分(=新たにクエリする必要がない)かどうかを表すフラグ
-			let enough = true;
-			if (options?._hint_?.emojis) {
-				for (const name of user.emojis) {
-					const matched = options._hint_.emojis.find(x => x.name === name && x.host === user.host);
-					if (matched) {
-						emojis.push(matched);
-					} else {
-						enough = false;
-					}
-				}
-			} else {
-				enough = false;
-			}
-
-			if (!enough) {
-				emojis = await Emojis.find({
-					where: {
-						name: In(user.emojis),
-						host: user.host
-					},
-					select: ['name', 'host', 'url', 'aliases']
-				});
-			}
-		}
-
 		const falsy = opts.detail ? false : undefined;
 
 		const packed = {
@@ -220,9 +189,7 @@ export class UserRepository extends Repository<User> {
 				faviconUrl: instance.faviconUrl,
 				themeColor: instance.themeColor,
 			} : undefined) : undefined,
-
+			emojis: populateEmojis(user.emojis, user.host),
-			// カスタム絵文字添付
-			emojis: emojis,
 
 			...(opts.detail ? {
 				url: profile!.url,