enhance(server): 画像圧縮周り（主にサムネイルの仕様）の変更 (#10287)

* DriveService, is-mime-image

* static, previewをavifに, アニメーション画像でもthumbnailを生成

* fallback

* animated: true

* fix

* avatarはwebp

* revert ?? file.url

---------

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

packages/backend/src/core/DriveService.ts

@@ -2,6 +2,7 @@ import * as fs from 'node:fs';
 import { Inject, Injectable } from '@nestjs/common';
 import { v4 as uuid } from 'uuid';
 import sharp from 'sharp';
+import { sharpBmp } from 'sharp-read-bmp';
 import { IsNull } from 'typeorm';
 import { DI } from '@/di-symbols.js';
 import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
@@ -34,6 +35,7 @@ import { FileInfoService } from '@/core/FileInfoService.js';
 import { bindThis } from '@/decorators.js';
 import { RoleService } from '@/core/RoleService.js';
 import { correctFilename } from '@/misc/correct-filename.js';
+import { isMimeImage } from '@/misc/is-mime-image.js';
 import type S3 from 'aws-sdk/clients/s3.js';
 
 type AddFileArgs = {
@@ -274,8 +276,8 @@ export class DriveService {
 			}
 		}
 
-		if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) {
+		if (!isMimeImage(type, 'sharp-convertible-image-with-bmp')) {
-			this.registerLogger.debug('web image and thumbnail not created (not an required file)');
+			this.registerLogger.debug('web image and thumbnail not created (cannot convert by sharp)');
 			return {
 				webpublic: null,
 				thumbnail: null,
@@ -284,22 +286,16 @@ export class DriveService {
 
 		let img: sharp.Sharp | null = null;
 		let satisfyWebpublic: boolean;
+		let isAnimated: boolean;
 
 		try {
-			img = sharp(path);
+			img = await sharpBmp(path, type);
 			const metadata = await img.metadata();
-			const isAnimated = metadata.pages && metadata.pages > 1;
+			isAnimated = !!(metadata.pages && metadata.pages > 1);
-
-			// skip animated
-			if (isAnimated) {
-				return {
-					webpublic: null,
-					thumbnail: null,
-				};
-			}
 
 			satisfyWebpublic = !!(
-				type !== 'image/svg+xml' && type !== 'image/avif' &&
+				type !== 'image/svg+xml' && // security reason
+				type !== 'image/avif' && // not supported by Mastodon
 			!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
 			metadata.width && metadata.width <= 2048 &&
 			metadata.height && metadata.height <= 2048
@@ -315,15 +311,13 @@ export class DriveService {
 		// #region webpublic
 		let webpublic: IImage | null = null;
 
-		if (generateWeb && !satisfyWebpublic) {
+		if (generateWeb && !satisfyWebpublic && !isAnimated) {
 			this.registerLogger.info('creating web image');
 
 			try {
-				if (type === 'image/jpeg') {
+				if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
-					webpublic = await this.imageProcessingService.convertSharpToJpeg(img, 2048, 2048);
-				} else if (['image/webp', 'image/avif'].includes(type)) {
 					webpublic = await this.imageProcessingService.convertSharpToWebp(img, 2048, 2048);
-				} else if (['image/png', 'image/svg+xml'].includes(type)) {
+				} else if (['image/png', 'image/bmp', 'image/svg+xml'].includes(type)) {
 					webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
 				} else {
 					this.registerLogger.debug('web image not created (not an required image)');
@@ -333,6 +327,7 @@ export class DriveService {
 			}
 		} else {
 			if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)');
+			else if (isAnimated) this.registerLogger.info('web image not created (animated image)');
 			else this.registerLogger.info('web image not created (from remote)');
 		}
 		// #endregion webpublic
@@ -341,10 +336,10 @@ export class DriveService {
 		let thumbnail: IImage | null = null;
 
 		try {
-			if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) {
+			if (isAnimated) {
-				thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 280);
+				thumbnail = await this.imageProcessingService.convertSharpToWebp(sharp(path, { animated: true }), 374, 317, { alphaQuality: 70 });
 			} else {
-				this.registerLogger.debug('thumbnail not created (not an required file)');
+				thumbnail = await this.imageProcessingService.convertSharpToAvif(img, 498, 422);
 			}
 		} catch (err) {
 			this.registerLogger.warn('thumbnail not created (an error occured)', err as Error);

packages/backend/src/core/ImageProcessingService.ts

@@ -15,15 +15,28 @@ export type IImageStream = {
 	type: string;
 };
 
-export type IImageStreamable = IImage | IImageStream;
+export type IImageSharp = {
+	data: sharp.Sharp;
+	ext: string | null;
+	type: string;
+};
+
+export type IImageStreamable = IImage | IImageStream | IImageSharp;
 
 export const webpDefault: sharp.WebpOptions = {
-	quality: 85,
+	quality: 77,
 	alphaQuality: 95,
 	lossless: false,
 	nearLossless: false,
 	smartSubsample: true,
 	mixed: true,
+	effort: 2,
+};
+
+export const avifDefault: sharp.AvifOptions = {
+	quality: 60,
+	lossless: false,
+	effort: 2,
 };
 
 import { bindThis } from '@/decorators.js';
@@ -37,36 +50,6 @@ export class ImageProcessingService {
 	) {
 	}
 
-	/**
-	 * Convert to JPEG
-	 *   with resize, remove metadata, resolve orientation, stop animation
-	 */
-	@bindThis
-	public async convertToJpeg(path: string, width: number, height: number): Promise<IImage> {
-		return this.convertSharpToJpeg(await sharp(path), width, height);
-	}
-
-	@bindThis
-	public async convertSharpToJpeg(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
-		const data = await sharp
-			.resize(width, height, {
-				fit: 'inside',
-				withoutEnlargement: true,
-			})
-			.rotate()
-			.jpeg({
-				quality: 85,
-				progressive: true,
-			})
-			.toBuffer();
-
-		return {
-			data,
-			ext: 'jpg',
-			type: 'image/jpeg',
-		};
-	}
-
 	/**
 	 * Convert to WebP
 	 *   with resize, remove metadata, resolve orientation, stop animation
@@ -78,29 +61,22 @@ export class ImageProcessingService {
 
 	@bindThis
 	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
-		const data = await sharp
+		const result = this.convertSharpToWebpStream(sharp, width, height, options);
-			.resize(width, height, {
-				fit: 'inside',
-				withoutEnlargement: true,
-			})
-			.rotate()
-			.webp(options)
-			.toBuffer();
 
 		return {
-			data,
+			data: await result.data.toBuffer(),
-			ext: 'webp',
+			ext: result.ext,
-			type: 'image/webp',
+			type: result.type,
 		};
 	}
 
 	@bindThis
-	public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream {
+	public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp {
 		return this.convertSharpToWebpStream(sharp(path), width, height, options);
 	}
 
 	@bindThis
-	public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream {
+	public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp {
 		const data = sharp
 			.resize(width, height, {
 				fit: 'inside',
@@ -115,13 +91,56 @@ export class ImageProcessingService {
 			type: 'image/webp',
 		};
 	}
+
+	/**
+	 * Convert to Avif
+	 *   with resize, remove metadata, resolve orientation, stop animation
+	 */
+	@bindThis
+	public async convertToAvif(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise<IImage> {
+		return this.convertSharpToAvif(sharp(path), width, height, options);
+	}
+
+	@bindThis
+	public async convertSharpToAvif(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise<IImage> {
+		const result = this.convertSharpToAvifStream(sharp, width, height, options);
+
+		return {
+			data: await result.data.toBuffer(),
+			ext: result.ext,
+			type: result.type,
+		};
+	}
+
+	@bindThis
+	public convertToAvifStream(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp {
+		return this.convertSharpToAvifStream(sharp(path), width, height, options);
+	}
+
+	@bindThis
+	public convertSharpToAvifStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp {
+		const data = sharp
+			.resize(width, height, {
+				fit: 'inside',
+				withoutEnlargement: true,
+			})
+			.rotate()
+			.avif(options);
+
+		return {
+			data,
+			ext: 'avif',
+			type: 'image/avif',
+		};
+	}
+
 	/**
 	 * Convert to PNG
 	 *   with resize, remove metadata, resolve orientation, stop animation
 	 */
 	@bindThis
 	public async convertToPng(path: string, width: number, height: number): Promise<IImage> {
-		return this.convertSharpToPng(await sharp(path), width, height);
+		return this.convertSharpToPng(sharp(path), width, height);
 	}
 
 	@bindThis

packages/backend/src/core/entities/DriveFileEntityService.ts

@@ -76,7 +76,7 @@ export class DriveFileEntityService {
 	@bindThis
 	private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string {
 		return appendQuery(
-			`${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
+			`${this.config.mediaProxy}/${mode ?? 'image'}.${mode === 'avatar' ? 'webp' : 'avif'}`,
 			query({
 				url,
 				...(mode ? { [mode]: '1' } : {}),
@@ -104,7 +104,7 @@ export class DriveFileEntityService {
 
 		const url = file.webpublicUrl ?? file.url;
 
-		return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null);
+		return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? url : null);
 	}
 
 	@bindThis

packages/backend/src/misc/is-mime-image.ts

@@ -2,10 +2,10 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 
 const dictionary = {
 	'safe-file': FILE_TYPE_BROWSERSAFE,
-	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
+	'sharp-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
-	'sharp-animation-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'],
+	'sharp-animation-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'],
-	'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
+	'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
-	'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
+	'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
 };
 
 export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);

packages/backend/src/server/FileServerService.ts

@@ -130,7 +130,7 @@ export class FileServerService {
 					if (isMimeImage(file.mime, 'sharp-convertible-image-with-bmp')) {
 						reply.header('Cache-Control', 'max-age=31536000, immutable');
 
-						const url = new URL(`${this.config.mediaProxy}/static.webp`);
+						const url = new URL(`${this.config.mediaProxy}/static.avif`);
 						url.searchParams.set('url', file.url);
 						url.searchParams.set('static', '1');
 
@@ -151,7 +151,7 @@ export class FileServerService {
 					if (['image/svg+xml'].includes(file.mime)) {
 						reply.header('Cache-Control', 'max-age=31536000, immutable');
 
-						const url = new URL(`${this.config.mediaProxy}/svg.webp`);
+						const url = new URL(`${this.config.mediaProxy}/svg.avif`);
 						url.searchParams.set('url', file.url);
 
 						file.cleanup();
@@ -291,9 +291,9 @@ export class FileServerService {
 					};
 				}
 			} else if ('static' in request.query) {
-				image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 498, 280);
+				image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 498, 422);
 			} else if ('preview' in request.query) {
-				image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 200, 200);
+				image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 200, 200);
 			} else if ('badge' in request.query) {
 				const mask = (await sharpBmp(file.path, file.mime))
 					.resize(96, 96, {
@@ -325,7 +325,7 @@ export class FileServerService {
 					type: 'image/png',
 				};
 			} else if (file.mime === 'image/svg+xml') {
-				image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048);
+				image = this.imageProcessingService.convertToAvifStream(file.path, 2048, 2048);
 			} else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) {
 				throw new StatusError('Rejected type', 403, 'Rejected type');
 			}

packages/backend/src/server/web/UrlPreviewService.ts

@@ -33,7 +33,7 @@ export class UrlPreviewService {
 	private wrap(url?: string | null): string | null {
 		return url != null
 			? url.match(/^https?:\/\//)
-				? `${this.config.mediaProxy}/preview.webp?${query({
+				? `${this.config.mediaProxy}/preview.avif?${query({
 					url,
 					preview: '1',
 				})}`

packages/frontend/src/components/MkMediaImage.vue

@@ -43,7 +43,7 @@ let darkMode = $ref(defaultStore.state.darkMode);
 const url = (props.raw || defaultStore.state.loadRawImages)
 	? props.image.url
 	: defaultStore.state.disableShowingAnimatedImages
-		? getStaticImageUrl(props.image.thumbnailUrl)
+		? getStaticImageUrl(props.image.url)
 		: props.image.thumbnailUrl;
 
 // Plugin:register_note_view_interruptor を使って書き換えられる可能性があるためwatchする

packages/frontend/src/pages/user/index.photos.vue

@@ -41,7 +41,7 @@ let images = $ref<{
 
 function thumbnail(image: misskey.entities.DriveFile): string {
 	return defaultStore.state.disableShowingAnimatedImages
-		? getStaticImageUrl(image.thumbnailUrl)
+		? getStaticImageUrl(image.url)
 		: image.thumbnailUrl;
 }
 

packages/frontend/src/scripts/media-proxy.ts

@@ -10,7 +10,10 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview', mustOrigi
 		imageUrl = (new URL(imageUrl)).searchParams.get('url') ?? imageUrl;
 	}
 
-	return `${mustOrigin ? localProxy : instance.mediaProxy}/image.webp?${query({
+	return `${mustOrigin ? localProxy : instance.mediaProxy}/${
+		type === 'preview' ? 'preview.avif'
+		: 'image.webp'
+	}?${query({
 		url: imageUrl,
 		fallback: '1',
 		...(type ? { [type]: '1' } : {}),
@@ -38,7 +41,7 @@ export function getStaticImageUrl(baseUrl: string): string {
 		return u.href;
 	}
 
-	return `${instance.mediaProxy}/static.webp?${query({
+	return `${instance.mediaProxy}/static.avif?${query({
 		url: u.href,
 		static: '1',
 	})}`;

packages/frontend/src/widgets/WidgetPhotos.vue

@@ -67,7 +67,7 @@ const onDriveFileCreated = (file) => {
 
 const thumbnail = (image: any): string => {
 	return defaultStore.state.disableShowingAnimatedImages
-		? getStaticImageUrl(image.thumbnailUrl)
+		? getStaticImageUrl(image.url)
 		: image.thumbnailUrl;
 };