perf(server): improvement of external mediaProxy (#9787)

* perf(server): improvement of external mediaProxy

* add a comment

* ✌️

* /filesでsharpの処理を行わずリダイレクトする

* fix

* thumbnail => static

* Fix #9788

* add avatar mode

* add url

* fix

* static.webp

* remove encodeURIComponent from media proxy path

* remove existance check

.config/example.yml

@@ -130,6 +130,7 @@ proxyBypassHosts:
 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
 
 # Media Proxy
+# Reference Implementation: https://github.com/misskey-dev/media-proxy
 #mediaProxy: https://example.com/proxy
 
 # Proxy remote files (default: false)

CHANGELOG.md

@@ -24,6 +24,9 @@ You should also include the user name that made the change.
 - syslogのサポートが削除されました
 
 ### Improvements
+- 外部メディアプロキシへの対応を強化しました  
+  外部メディアプロキシのFastify実装を作りました  
+  https://github.com/misskey-dev/media-proxy
 - ロールで広告の非表示が有効になっている場合は最初から広告を非表示にするように
 
 ## 13.2.6 (2023/02/01)

packages/backend/src/config.ts

@@ -87,6 +87,8 @@ export type Mixin = {
 	userAgent: string;
 	clientEntry: string;
 	clientManifestExists: boolean;
+	mediaProxy: string;
+	externalMediaProxyEnabled: boolean;
 };
 
 export type Config = Source & Mixin;
@@ -135,6 +137,13 @@ export function loadConfig() {
 	mixin.clientEntry = clientManifest['src/init.ts'];
 	mixin.clientManifestExists = clientManifestExists;
 
+	const externalMediaProxy = config.mediaProxy ?
+		config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy
+		: null;
+	const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`;
+	mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy;
+	mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
+
 	if (!config.redis.prefix) config.redis.prefix = mixin.host;
 
 	return Object.assign(config, mixin);

packages/backend/src/core/CustomEmojiService.ts

@@ -120,7 +120,7 @@ export class CustomEmojiService {
 		const url = isLocal
 			? emojiUrl
 			: this.config.proxyRemoteFiles
-				? `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`
+				? `${this.config.mediaProxy}/emoji.webp?${query({ url: emojiUrl })}`
 				: emojiUrl;
 
 		return url;

packages/backend/src/core/entities/ChannelEntityService.ts

@@ -54,7 +54,7 @@ export class ChannelEntityService {
 			name: channel.name,
 			description: channel.description,
 			userId: channel.userId,
-			bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner, false) : null,
+			bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner) : null,
 			usersCount: channel.usersCount,
 			notesCount: channel.notesCount,
 

packages/backend/src/core/entities/DriveFileEntityService.ts

@@ -71,27 +71,41 @@ export class DriveFileEntityService {
 	}
 
 	@bindThis
-	public getPublicUrl(file: DriveFile, thumbnail = false): string | null {
+	public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail
+		const proxiedUrl = (url: string) => appendQuery(
+			`${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
+			query({
+				url,
+				...(mode ? { [mode]: '1' } : {}),
+			})
+		);
+
 		// リモートかつメディアプロキシ
-		if (file.uri != null && file.userHost != null && this.config.mediaProxy != null) {
-			return appendQuery(this.config.mediaProxy, query({
-				url: file.uri,
-				thumbnail: thumbnail ? '1' : undefined,
-			}));
+		if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
+			return proxiedUrl(file.uri);
 		}
 
 		// リモートかつ期限切れはローカルプロキシを試みる
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
-			const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey;
+			const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey;
 
 			if (key && !key.match('/')) {	// 古いものはここにオブジェクトストレージキーが入ってるので除外
-				return `${this.config.url}/files/${key}`;
+				const url = `${this.config.url}/files/${key}`;
+				if (mode === 'avatar') return proxiedUrl(url);
+				return url;
 			}
 		}
 
 		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type);
 
-		return thumbnail ? (file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null)) : (file.webpublicUrl ?? file.url);
+		if (mode === 'static') {
+			return file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null);
+		}
+
+		const url = file.webpublicUrl ?? file.url;
+
+		if (mode === 'avatar') return proxiedUrl(url);
+		return url;
 	}
 
 	@bindThis
@@ -166,8 +180,8 @@ export class DriveFileEntityService {
 			isSensitive: file.isSensitive,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
-			url: opts.self ? file.url : this.getPublicUrl(file, false),
-			thumbnailUrl: this.getPublicUrl(file, true),
+			url: opts.self ? file.url : this.getPublicUrl(file),
+			thumbnailUrl: this.getPublicUrl(file, 'static'),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
@@ -201,8 +215,8 @@ export class DriveFileEntityService {
 			isSensitive: file.isSensitive,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
-			url: opts.self ? file.url : this.getPublicUrl(file, false),
-			thumbnailUrl: this.getPublicUrl(file, true),
+			url: opts.self ? file.url : this.getPublicUrl(file),
+			thumbnailUrl: this.getPublicUrl(file, 'static'),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {

packages/backend/src/core/entities/UserEntityService.ts

@@ -314,10 +314,10 @@ export class UserEntityService implements OnModuleInit {
 	@bindThis
 	public async getAvatarUrl(user: User): Promise<string> {
 		if (user.avatar) {
-			return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else if (user.avatarId) {
 			const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId });
-			return this.driveFileEntityService.getPublicUrl(avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else {
 			return this.getIdenticonUrl(user.id);
 		}
@@ -326,7 +326,7 @@ export class UserEntityService implements OnModuleInit {
 	@bindThis
 	public getAvatarUrlSync(user: User): string {
 		if (user.avatar) {
-			return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else {
 			return this.getIdenticonUrl(user.id);
 		}
@@ -422,7 +422,7 @@ export class UserEntityService implements OnModuleInit {
 				createdAt: user.createdAt.toISOString(),
 				updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
 				lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null,
-				bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner, false) : null,
+				bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner) : null,
 				bannerBlurhash: user.banner?.blurhash ?? null,
 				isLocked: user.isLocked,
 				isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote),

packages/backend/src/server/FileServerService.ts

@@ -137,38 +137,38 @@ export class FileServerService {
 
 		try {
 			if (file.state === 'remote') {
-				const convertFile = async () => {
+				let image: IImageStreamable | null = null;
+
 				if (file.fileRole === 'thumbnail') {
-						if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(file.mime)) {
-							return this.imageProcessingService.convertToWebpStream(
-								file.path,
-								498,
-								280
-							);
+					if (isMimeImage(file.mime, 'sharp-convertible-image')) {
+						reply.header('Cache-Control', 'max-age=31536000, immutable');
+
+						const url = new URL(`${this.config.mediaProxy}/static.webp`);
+						url.searchParams.set('url', file.url);
+						url.searchParams.set('static', '1');
+						return await reply.redirect(301, url.toString());
 					} else if (file.mime.startsWith('video/')) {
-							return await this.videoProcessingService.generateVideoThumbnail(file.path);
+						image = await this.videoProcessingService.generateVideoThumbnail(file.path);
 					}
 				}
 
 				if (file.fileRole === 'webpublic') {
 					if (['image/svg+xml'].includes(file.mime)) {
-							return this.imageProcessingService.convertToWebpStream(
-								file.path,
-								2048,
-								2048,
-								{ ...webpDefault, lossless: true }
-							)
+						reply.header('Cache-Control', 'max-age=31536000, immutable');
+
+						const url = new URL(`${this.config.mediaProxy}/svg.webp`);
+						url.searchParams.set('url', file.url);
+						return await reply.redirect(301, url.toString());
 					}
 				}
 
-					return {
+				if (!image) {
+					image = {
 						data: fs.createReadStream(file.path),
 						ext: file.ext,
 						type: file.mime,
 					};
-				};
-
-				const image = await convertFile();
+				}
 
 				if ('pipe' in image.data && typeof image.data.pipe === 'function') {
 					// image.dataがstreamなら、stream終了後にcleanup
@@ -180,7 +180,6 @@ export class FileServerService {
 				}
 
 				reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(image.type) ? image.type : 'application/octet-stream');
-				reply.header('Cache-Control', 'max-age=31536000, immutable');
 				return image.data;
 			}
 
@@ -217,6 +216,23 @@ export class FileServerService {
 			return;
 		}
 
+		if (this.config.externalMediaProxyEnabled) {
+			// 外部のメディアプロキシが有効なら、そちらにリダイレクト
+
+			reply.header('Cache-Control', 'public, max-age=259200'); // 3 days
+
+			const url = new URL(`${this.config.mediaProxy}/${request.params.url || ''}`);
+
+			for (const [key, value] of Object.entries(request.query)) {
+				url.searchParams.append(key, value);
+			}
+
+			return await reply.redirect(
+				301,
+				url.toString(),
+			);
+		}
+
 		// Create temp file
 		const file = await this.getStreamAndTypeFromUrl(url);
 		if (file === '404') {
@@ -236,7 +252,7 @@ export class FileServerService {
 			const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image');
 
 			let image: IImageStreamable | null = null;
-			if ('emoji' in request.query && isConvertibleImage) {
+			if (('emoji' in request.query || 'avatar' in request.query) && isConvertibleImage) {
 				if (!isAnimationConvertibleImage && !('static' in request.query)) {
 					image = {
 						data: fs.createReadStream(file.path),
@@ -246,7 +262,7 @@ export class FileServerService {
 				} else {
 					const data = sharp(file.path, { animated: !('static' in request.query) })
 							.resize({
-								height: 128,
+								height: 'emoji' in request.query ? 128 : 320,
 								withoutEnlargement: true,
 							})
 							.webp(webpDefault);
@@ -370,7 +386,7 @@ export class FileServerService {
 
 	@bindThis
 	private async getFileFromKey(key: string): Promise<
-		{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; }
+		{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; url: string; mime: string; ext: string | null; path: string; cleanup: () => void; }
 		| { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; }
 		| '404'
 		| '204'
@@ -392,6 +408,7 @@ export class FileServerService {
 			const result = await this.downloadAndDetectTypeFromUrl(file.uri);
 			return {
 				...result,
+				url: file.uri,
 				fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original',
 				file,
 			}

packages/backend/src/server/ServerService.ts

@@ -106,7 +106,7 @@ export class ServerService {
 				}
 			}
 
-			const url = new URL('/proxy/emoji.webp', this.config.url);
+			const url = new URL(`${this.config.mediaProxy}/emoji.webp`);
 			// || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
 			url.searchParams.set('url', emoji.publicUrl || emoji.originalUrl);
 			url.searchParams.set('emoji', '1');

packages/backend/src/server/api/endpoints/meta.ts

@@ -181,6 +181,10 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			mediaProxy: {
+				type: 'string',
+				optional: false, nullable: false,
+			},
 			features: {
 				type: 'object',
 				optional: true, nullable: false,
@@ -307,6 +311,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },
 
+				mediaProxy: this.config.mediaProxy,
+
 				...(ps.detail ? {
 					pinnedPages: instance.pinnedPages,
 					pinnedClipId: instance.pinnedClipId,

packages/backend/src/server/web/UrlPreviewService.ts

@@ -33,7 +33,7 @@ export class UrlPreviewService {
 	private wrap(url?: string): string | null {
 		return url != null
 			? url.match(/^https?:\/\//)
-				? `${this.config.url}/proxy/preview.webp?${query({
+				? `${this.config.mediaProxy}/preview.webp?${query({
 					url,
 					preview: '1',
 				})}`

packages/frontend/src/scripts/media-proxy.ts

@@ -1,8 +1,9 @@
 import { query, appendQuery } from '@/scripts/url';
 import { url } from '@/config';
+import { instance } from '@/instance';
 
 export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
-	if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) {
+	if (imageUrl.startsWith(instance.mediaProxy + '/') || imageUrl.startsWith('/proxy/')) {
 		// もう既にproxyっぽそうだったらsearchParams付けるだけ
 		return appendQuery(imageUrl, query({
 			fallback: '1',
@@ -10,7 +11,7 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
 		}));
 	}
 
-	return `${url}/proxy/image.webp?${query({
+	return `${instance.mediaProxy}/image.webp?${query({
 		url: imageUrl,
 		fallback: '1',
 		...(type ? { [type]: '1' } : {}),
@@ -25,22 +26,19 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined,
 export function getStaticImageUrl(baseUrl: string): string {
 	const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url);
 
-	if (u.href.startsWith(`${url}/proxy/`)) {
-		// もう既にproxyっぽそうだったらsearchParams付けるだけ
-		u.searchParams.set('static', '1');
-		return u.href;
-	}
-
 	if (u.href.startsWith(`${url}/emoji/`)) {
 		// もう既にemojiっぽそうだったらsearchParams付けるだけ
 		u.searchParams.set('static', '1');
 		return u.href;
 	}
 
-	// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
-	const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
+	if (u.href.startsWith(instance.mediaProxy + '/')) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+		u.searchParams.set('static', '1');
+		return u.href;
+	}
 
-	return `${url}/proxy/${dummy}?${query({
+	return `${instance.mediaProxy}/static.webp?${query({
 		url: u.href,
 		static: '1',
 	})}`;