feat: カスタム絵文字エクスポート

CHANGELOG.md

@@ -13,6 +13,7 @@
 - Added a user-level instance mute in user settings
 - フォローエクスポートでミュートしているユーザーを含めないオプションを追加
 - フォローエクスポートで使われていないアカウントを含めないオプションを追加
+- カスタム絵文字エクスポート機能
 
 ### Bugfixes
 - クライアント: タッチ機能付きディスプレイを使っていてマウス操作をしている場合に一部機能が動作しない問題を修正

packages/backend/package.json

@@ -74,6 +74,7 @@
 		"@typescript-eslint/eslint-plugin": "5.3.1",
 		"@typescript-eslint/parser": "5.1.0",
 		"abort-controller": "3.0.0",
+		"archiver": "5.3.0",
 		"autobind-decorator": "2.4.0",
 		"autosize": "4.0.4",
 		"autwh": "0.1.0",
@@ -131,6 +132,7 @@
 		"koa-views": "7.0.2",
 		"langmap": "0.0.16",
 		"mfm-js": "0.20.0",
+		"mime-types": "2.1.34",
 		"misskey-js": "0.0.8",
 		"mocha": "8.4.0",
 		"ms": "3.0.0-canary.1",

packages/backend/src/misc/download-url.ts

@@ -11,7 +11,7 @@ const PrivateIp = require('private-ip');
 
 const pipeline = util.promisify(stream.pipeline);
 
-export async function downloadUrl(url: string, path: string) {
+export async function downloadUrl(url: string, path: string): Promise<void> {
 	const logger = new Logger('download');
 
 	logger.info(`Downloading ${chalk.cyan(url)} ...`);

packages/backend/src/queue/index.ts

@@ -117,6 +117,15 @@ export function createDeleteDriveFilesJob(user: ThinUser) {
 	});
 }
 
+export function createExportCustomEmojisJob(user: ThinUser) {
+	return dbQueue.add('exportCustomEmojis', {
+		user: user,
+	}, {
+		removeOnComplete: true,
+		removeOnFail: true,
+	});
+}
+
 export function createExportNotesJob(user: ThinUser) {
 	return dbQueue.add('exportNotes', {
 		user: user,

packages/backend/src/queue/processors/db/export-custom-emojis.ts

@@ -0,0 +1,131 @@
+import * as Bull from 'bull';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+
+import { ulid } from 'ulid';
+const mime = require('mime-types');
+const archiver = require('archiver');
+import { queueLogger } from '../../logger';
+import addFile from '@/services/drive/add-file';
+import * as dateFormat from 'dateformat';
+import { Users, Emojis } from '@/models/index';
+import {  } from '@/queue/types';
+import { downloadUrl } from '@/misc/download-url';
+
+const logger = queueLogger.createSubLogger('export-custom-emojis');
+
+export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> {
+	logger.info(`Exporting custom emojis ...`);
+
+	const user = await Users.findOne(job.data.user.id);
+	if (user == null) {
+		done();
+		return;
+	}
+
+	// Create temp dir
+	const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.dir((e, path, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+
+	logger.info(`Temp dir is ${path}`);
+
+	const metaPath = path + '/meta.json';
+
+	fs.writeFileSync(metaPath, '', 'utf-8');
+
+	const metaStream = fs.createWriteStream(metaPath, { flags: 'a' });
+
+	await new Promise<void>((res, rej) => {
+		metaStream.write('[', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	const customEmojis = await Emojis.find({
+		where: {
+			host: null,
+		},
+		order: {
+			id: 'ASC',
+		},
+	});
+
+	for (const emoji of customEmojis) {
+		const exportId = ulid().toLowerCase();
+		const emojiPath = path + '/' + exportId + '.' + mime.extension(emoji.type);
+		fs.writeFileSync(emojiPath, '', 'binary');
+		let downloaded = false;
+
+		try {
+			await downloadUrl(emoji.url, emojiPath);
+			downloaded = true;
+		} catch (e) { // TODO: 何度か再試行
+			logger.error(e);
+		}
+
+		await new Promise<void>((res, rej) => {
+			const content = JSON.stringify({
+				id: exportId,
+				downloaded: downloaded,
+				emoji: emoji,
+			});
+			const isFirst = customEmojis.indexOf(emoji) === 0;
+			metaStream.write(isFirst ? content : ',\n' + content, err => {
+				if (err) {
+					logger.error(err);
+					rej(err);
+				} else {
+					res();
+				}
+			});
+		});
+	}
+
+	await new Promise<void>((res, rej) => {
+		metaStream.write(']', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	metaStream.end();
+
+	// Create archive
+	const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.file((e, path, fd, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+	const archiveStream = fs.createWriteStream(archivePath);
+	const archive = archiver('zip', {
+		zlib: { level: 0 },
+	});
+	archiveStream.on('close', async () => {
+		logger.succ(`Exported to: ${archivePath}`);
+
+		const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip';
+		const driveFile = await addFile(user, archivePath, fileName, null, null, true);
+
+		logger.succ(`Exported to: ${driveFile.id}`);
+		cleanup();
+		archiveCleanup();
+		done();
+	});
+	archive.pipe(archiveStream);
+	archive.directory(path, false);
+	archive.finalize();
+}

packages/backend/src/queue/processors/db/export-notes.ts

@@ -74,7 +74,8 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
 			}
 			const content = JSON.stringify(serialize(note, poll));
 			await new Promise<void>((res, rej) => {
-				stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
+				const isFirst = exportedNotesCount === 0;
+				stream.write(isFirst ? content : ',\n' + content, err => {
 					if (err) {
 						logger.error(err);
 						rej(err);

packages/backend/src/queue/processors/db/index.ts

@@ -1,6 +1,7 @@
 import * as Bull from 'bull';
 import { DbJobData } from '@/queue/types';
 import { deleteDriveFiles } from './delete-drive-files';
+import { exportCustomEmojis } from './export-custom-emojis';
 import { exportNotes } from './export-notes';
 import { exportFollowing } from './export-following';
 import { exportMute } from './export-mute';
@@ -14,6 +15,7 @@ import { importBlocking } from './import-blocking';
 
 const jobs = {
 	deleteDriveFiles,
+	exportCustomEmojis,
 	exportNotes,
 	exportFollowing,
 	exportMute,

packages/backend/src/server/api/endpoints/export-custom-emojis.ts

@@ -0,0 +1,17 @@
+import $ from 'cafy';
+import define from '../define';
+import { createExportCustomEmojisJob } from '@/queue/index';
+import ms from 'ms';
+
+export const meta = {
+	secure: true,
+	requireCredential: true as const,
+	limit: {
+		duration: ms('1hour'),
+		max: 1,
+	},
+};
+
+export default define(meta, async (ps, user) => {
+	createExportCustomEmojisJob(user);
+});

packages/backend/yarn.lock

@@ -1318,6 +1318,35 @@ aproba@^1.0.3:
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
   integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==
 
+archiver-utils@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2"
+  integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==
+  dependencies:
+    glob "^7.1.4"
+    graceful-fs "^4.2.0"
+    lazystream "^1.0.0"
+    lodash.defaults "^4.2.0"
+    lodash.difference "^4.5.0"
+    lodash.flatten "^4.4.0"
+    lodash.isplainobject "^4.0.6"
+    lodash.union "^4.6.0"
+    normalize-path "^3.0.0"
+    readable-stream "^2.0.0"
+
+archiver@5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba"
+  integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg==
+  dependencies:
+    archiver-utils "^2.1.0"
+    async "^3.2.0"
+    buffer-crc32 "^0.2.1"
+    readable-stream "^3.6.0"
+    readdir-glob "^1.0.0"
+    tar-stream "^2.2.0"
+    zip-stream "^4.1.0"
+
 are-we-there-yet@~1.1.2:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21"
@@ -1645,6 +1674,11 @@ buffer-alloc@^1.2.0:
     buffer-alloc-unsafe "^1.1.0"
     buffer-fill "^1.0.0"
 
+buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
+  version "0.2.13"
+  resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
+  integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=
+
 buffer-equal-constant-time@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
@@ -2167,6 +2201,16 @@ compare-versions@3.6.0:
   resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62"
   integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==
 
+compress-commons@^4.1.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.1.tgz#df2a09a7ed17447642bad10a85cc9a19e5c42a7d"
+  integrity sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==
+  dependencies:
+    buffer-crc32 "^0.2.13"
+    crc32-stream "^4.0.2"
+    normalize-path "^3.0.0"
+    readable-stream "^3.6.0"
+
 concat-map@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -2249,7 +2293,7 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
-crc-32@1.2.0:
+crc-32@1.2.0, crc-32@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
   integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
@@ -2257,6 +2301,14 @@ crc-32@1.2.0:
     exit-on-epipe "~1.0.1"
     printj "~1.1.0"
 
+crc32-stream@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-4.0.2.tgz#c922ad22b38395abe9d3870f02fa8134ed709007"
+  integrity sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==
+  dependencies:
+    crc-32 "^1.2.0"
+    readable-stream "^3.4.0"
+
 create-require@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
@@ -4901,6 +4953,13 @@ langmap@0.0.16:
   resolved "https://registry.yarnpkg.com/langmap/-/langmap-0.0.16.tgz#2fe3e98a531fec0fec546624ebe168c2855bab56"
   integrity sha512-AtYvBK7BsDvWwnSfmO7CfgeUy7GUT1wK3QX8eKH/Ey/eXodqoHuAtvdQ82hmWD9QVFVKnuiNjym9fGY4qSJeLA==
 
+lazystream@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.1.tgz#494c831062f1f9408251ec44db1cba29242a2638"
+  integrity sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==
+  dependencies:
+    readable-stream "^2.0.5"
+
 lcid@^3.0.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/lcid/-/lcid-3.1.1.tgz#9030ec479a058fc36b5e8243ebaac8b6ac582fd0"
@@ -4981,6 +5040,11 @@ lodash.defaults@^4.0.1, lodash.defaults@^4.2.0:
   resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
   integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
 
+lodash.difference@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
+  integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=
+
 lodash.filter@^4.4.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
@@ -5006,6 +5070,11 @@ lodash.isfinite@^3.3.2:
   resolved "https://registry.yarnpkg.com/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz#fb89b65a9a80281833f0b7478b3a5104f898ebb3"
   integrity sha1-+4m2WpqAKBgz8LdHizpRBPiY67M=
 
+lodash.isplainobject@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
+  integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
+
 lodash.isregexp@3.0.5:
   version "3.0.5"
   resolved "https://registry.yarnpkg.com/lodash.isregexp/-/lodash.isregexp-3.0.5.tgz#e0f596242f2fa228a840086b6c8ad82e4b71fd2d"
@@ -5051,6 +5120,11 @@ lodash.sortby@^4.7.0:
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
   integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
 
+lodash.union@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
+  integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
+
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@@ -5198,6 +5272,18 @@ mime-db@1.44.0:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
   integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==
 
+mime-db@1.51.0:
+  version "1.51.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c"
+  integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==
+
+mime-types@2.1.34:
+  version "2.1.34"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24"
+  integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==
+  dependencies:
+    mime-db "1.51.0"
+
 mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24:
   version "2.1.27"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
@@ -6777,7 +6863,7 @@ readable-stream@1.1.x:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@^2.0.6, readable-stream@^2.2.2:
+readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -6807,6 +6893,13 @@ readable-web-to-node-stream@^3.0.0:
     "@types/readable-stream" "^2.3.9"
     readable-stream "^3.6.0"
 
+readdir-glob@^1.0.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
+  integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==
+  dependencies:
+    minimatch "^3.0.4"
+
 readdirp@~3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.3.0.tgz#984458d13a1e42e2e9f5841b129e162f369aff17"
@@ -7664,7 +7757,7 @@ tar-stream@^2.0.0:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
-tar-stream@^2.1.4:
+tar-stream@^2.1.4, tar-stream@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
   integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
@@ -8583,3 +8676,12 @@ zen-observable@^0.8.15:
   version "0.8.15"
   resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
   integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==
+
+zip-stream@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79"
+  integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==
+  dependencies:
+    archiver-utils "^2.1.0"
+    compress-commons "^4.1.0"
+    readable-stream "^3.6.0"

packages/client/src/pages/emojis.vue

@@ -21,10 +21,38 @@ export default defineComponent({
 				title: this.$ts.customEmojis,
 				icon: 'fas fa-laugh',
 				bg: 'var(--bg)',
+				actions: [{
+					icon: 'fas fa-ellipsis-h',
+					handler: this.menu
+				}],
 			})),
 			tab: 'category',
 		}
 	},
+
+	methods: {
+		menu(ev) {
+			os.popupMenu([{
+				icon: 'fas fa-download',
+				text: this.$ts.export,
+				action: async () => {
+					os.api('export-custom-emojis', {
+					})
+					.then(() => {
+						os.alert({
+							type: 'info',
+							text: this.$ts.exportRequested,
+						});
+					}).catch((e) => {
+						os.alert({
+							type: 'error',
+							text: e.message,
+						});
+					});
+				}
+			}], ev.currentTarget || ev.target);
+		}
+	}
 });
 </script>