Improve drive management

locales/ja-JP.yml

@@ -1417,6 +1417,9 @@ admin/views/drive.vue:
   unmark-as-sensitive: "閲覧注意を解除"
   marked-as-sensitive: "閲覧注意に設定しました"
   unmarked-as-sensitive: "閲覧注意を解除しました"
+  clean-remote-files: "リモートファイルのキャッシュを削除"
+  clean-remote-files-are-you-sure: "すべてのリモートファイルのキャッシュを削除してもよろしいですか？"
+  clean-up: "クリーンアップ"
 
 admin/views/users.vue:
   operation: "操作"

src/client/app/admin/views/drive.vue

@@ -14,6 +14,10 @@
 			<ui-button @click="show()"><fa :icon="faSearch"/> {{ $t('lookup') }}</ui-button>
 			<ui-textarea v-if="file" :value="file | json5" readonly tall style="margin-top:16px;"></ui-textarea>
 		</section>
+		<section>
+			<ui-button @click="cleanUp()"><fa :icon="faTrashAlt"/> {{ $t('clean-up') }}</ui-button>
+			<ui-button @click="cleanRemoteFiles()"><fa :icon="faTrashAlt"/> {{ $t('clean-remote-files') }}</ui-button>
+		</section>
 	</ui-card>
 
 	<ui-card>
@@ -227,6 +231,29 @@ export default Vue.extend({
 				});
 			});
 		},
+
+		cleanRemoteFiles() {
+			this.$root.dialog({
+				type: 'warning',
+				text: this.$t('clean-remote-files-are-you-sure'),
+				showCancelButton: true
+			}).then(({ canceled }) => {
+				if (canceled) return;
+				this.$root.api('admin/drive/clean-remote-files');
+				this.$root.dialog({
+					type: 'success',
+					splash: true
+				});
+			});
+		},
+
+		cleanUp() {
+			this.$root.api('admin/drive/cleanup');
+			this.$root.dialog({
+				type: 'success',
+				splash: true
+			});
+		}
 	}
 });
 </script>

src/queue/index.ts

@@ -8,6 +8,7 @@ import { program } from '../argv';
 import processDeliver from './processors/deliver';
 import processInbox from './processors/inbox';
 import processDb from './processors/db';
+import procesObjectStorage from './processors/object-storage';
 import { queueLogger } from './logger';
 import { DriveFile } from '../models/entities/drive-file';
 
@@ -34,9 +35,12 @@ function renderError(e: Error): any {
 export const deliverQueue = initializeQueue('deliver');
 export const inboxQueue = initializeQueue('inbox');
 export const dbQueue = initializeQueue('db');
+export const objectStorageQueue = initializeQueue('objectStorage');
 
 const deliverLogger = queueLogger.createSubLogger('deliver');
 const inboxLogger = queueLogger.createSubLogger('inbox');
+const dbLogger = queueLogger.createSubLogger('db');
+const objectStorageLogger = queueLogger.createSubLogger('objectStorage');
 
 deliverQueue
 	.on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`))
@@ -54,6 +58,22 @@ inboxQueue
 	.on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) }))
 	.on('stalled', (job) => inboxLogger.warn(`stalled id=${job.id} activity=${job.data.activity ? job.data.activity.id : 'none'}`));
 
+dbQueue
+	.on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`))
+	.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
+	.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
+	.on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
+	.on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) }))
+	.on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`));
+
+objectStorageQueue
+	.on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`))
+	.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
+	.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
+	.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
+	.on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) }))
+	.on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`));
+
 export function deliver(user: ILocalUser, content: any, to: any) {
 	if (content == null) return null;
 
@@ -165,11 +185,21 @@ export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id
 	});
 }
 
+export function createDeleteObjectStorageFileJob(key: string) {
+	return objectStorageQueue.add('deleteFile', {
+		key: key
+	}, {
+		removeOnComplete: true,
+		removeOnFail: true
+	});
+}
+
 export default function() {
 	if (!program.onlyServer) {
 		deliverQueue.process(128, processDeliver);
 		inboxQueue.process(128, processInbox);
 		processDb(dbQueue);
+		procesObjectStorage(objectStorageQueue);
 	}
 }
 

src/queue/processors/db/delete-drive-files.ts

@@ -1,7 +1,7 @@
 import * as Bull from 'bull';
 
 import { queueLogger } from '../../logger';
-import deleteFile from '../../../services/drive/delete-file';
+import { deleteFile } from '../../../services/drive/delete-file';
 import { Users, DriveFiles } from '../../../models';
 import { MoreThan } from 'typeorm';
 

src/queue/processors/object-storage/delete-file.ts

@@ -0,0 +1,22 @@
+import * as Bull from 'bull';
+import * as Minio from 'minio';
+import { fetchMeta } from '../../../misc/fetch-meta';
+
+export default async (job: Bull.Job) => {
+	const meta = await fetchMeta();
+
+	const minio = new Minio.Client({
+		endPoint: meta.objectStorageEndpoint!,
+		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
+		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
+		useSSL: meta.objectStorageUseSSL,
+		accessKey: meta.objectStorageAccessKey!,
+		secretKey: meta.objectStorageSecretKey!,
+	});
+
+	const key: string = job.data.key;
+
+	await minio.removeObject(meta.objectStorageBucket!, key);
+
+	return 'Success';
+};

src/queue/processors/object-storage/index.ts

@@ -0,0 +1,12 @@
+import * as Bull from 'bull';
+import deleteFile from './delete-file';
+
+const jobs = {
+	deleteFile,
+} as any;
+
+export default function(q: Bull.Queue) {
+	for (const [k, v] of Object.entries(jobs)) {
+		q.process(k, v as any);
+	}
+}

src/server/api/endpoints/admin/delete-all-files-of-a-user.ts

@@ -1,6 +1,6 @@
 import $ from 'cafy';
 import define from '../../define';
-import del from '../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../services/drive/delete-file';
 import { DriveFiles } from '../../../../models';
 import { ID } from '../../../../misc/cafy-id';
 
@@ -27,6 +27,6 @@ export default define(meta, async (ps, me) => {
 	});
 
 	for (const file of files) {
-		del(file);
+		deleteFile(file);
 	}
 });

src/server/api/endpoints/admin/drive/clean-remote-files.ts

@@ -0,0 +1,21 @@
+import { Not, IsNull } from 'typeorm';
+import define from '../../../define';
+import { deleteFile } from '../../../../../services/drive/delete-file';
+import { DriveFiles } from '../../../../../models';
+
+export const meta = {
+	tags: ['admin'],
+
+	requireCredential: true,
+	requireModerator: true,
+};
+
+export default define(meta, async (ps, me) => {
+	const files = await DriveFiles.find({
+		userHost: Not(IsNull())
+	});
+
+	for (const file of files) {
+		deleteFile(file, true);
+	}
+});

src/server/api/endpoints/admin/drive/cleanup.ts

@@ -0,0 +1,21 @@
+import { IsNull } from 'typeorm';
+import define from '../../../define';
+import { deleteFile } from '../../../../../services/drive/delete-file';
+import { DriveFiles } from '../../../../../models';
+
+export const meta = {
+	tags: ['admin'],
+
+	requireCredential: true,
+	requireModerator: true,
+};
+
+export default define(meta, async (ps, me) => {
+	const files = await DriveFiles.find({
+		userId: IsNull()
+	});
+
+	for (const file of files) {
+		deleteFile(file);
+	}
+});

src/server/api/endpoints/admin/federation/delete-all-files.ts

@@ -1,6 +1,6 @@
 import $ from 'cafy';
 import define from '../../../define';
-import del from '../../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../../services/drive/delete-file';
 import { DriveFiles } from '../../../../../models';
 
 export const meta = {
@@ -22,6 +22,6 @@ export default define(meta, async (ps, me) => {
 	});
 
 	for (const file of files) {
-		del(file);
+		deleteFile(file);
 	}
 });

src/server/api/endpoints/drive/files/delete.ts

@@ -1,6 +1,6 @@
 import $ from 'cafy';
 import { ID } from '../../../../../misc/cafy-id';
-import del from '../../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../../services/drive/delete-file';
 import { publishDriveStream } from '../../../../../services/stream';
 import define from '../../../define';
 import { ApiError } from '../../../error';
@@ -57,7 +57,7 @@ export default define(meta, async (ps, user) => {
 	}
 
 	// Delete
-	await del(file);
+	await deleteFile(file);
 
 	// Publish fileDeleted event
 	publishDriveStream(user.id, 'fileDeleted', file.id);

src/services/drive/add-file.ts

@@ -7,7 +7,7 @@ import * as uuid from 'uuid';
 import * as sharp from 'sharp';
 
 import { publishMainStream, publishDriveStream } from '../stream';
-import delFile from './delete-file';
+import { deleteFile } from './delete-file';
 import { fetchMeta } from '../../misc/fetch-meta';
 import { GenerateVideoThumbnail } from './generate-video-thumbnail';
 import { driveLogger } from './logger';
@@ -233,7 +233,7 @@ async function deleteOldFile(user: IRemoteUser) {
 	const oldFile = await q.getOne();
 
 	if (oldFile) {
-		delFile(oldFile, true);
+		deleteFile(oldFile, true);
 	}
 }
 

src/services/drive/delete-file.ts

@@ -1,11 +1,10 @@
-import * as Minio from 'minio';
 import { DriveFile } from '../../models/entities/drive-file';
 import { InternalStorage } from './internal-storage';
 import { DriveFiles, Instances, Notes } from '../../models';
 import { driveChart, perUserDriveChart, instanceChart } from '../chart';
-import { fetchMeta } from '../../misc/fetch-meta';
+import { createDeleteObjectStorageFileJob } from '../../queue';
 
-export default async function(file: DriveFile, isExpired = false) {
+export async function deleteFile(file: DriveFile, isExpired = false) {
 	if (file.storedInternal) {
 		InternalStorage.del(file.accessKey!);
 
@@ -17,25 +16,14 @@ export default async function(file: DriveFile, isExpired = false) {
 			InternalStorage.del(file.webpublicAccessKey!);
 		}
 	} else if (!file.isLink) {
-		const meta = await fetchMeta();
-
-		const minio = new Minio.Client({
-			endPoint: meta.objectStorageEndpoint!,
-			region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
-			port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
-			useSSL: meta.objectStorageUseSSL,
-			accessKey: meta.objectStorageAccessKey!,
-			secretKey: meta.objectStorageSecretKey!,
-		});
-
-		await minio.removeObject(meta.objectStorageBucket!, file.accessKey!);
+		createDeleteObjectStorageFileJob(file.accessKey!);
 
 		if (file.thumbnailUrl) {
-			await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!);
+			createDeleteObjectStorageFileJob(file.thumbnailAccessKey!);
 		}
 
 		if (file.webpublicUrl) {
-			await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!);
+			createDeleteObjectStorageFileJob(file.webpublicAccessKey!);
 		}
 	}
 
@@ -44,8 +32,8 @@ export default async function(file: DriveFile, isExpired = false) {
 		DriveFiles.update(file.id, {
 			isLink: true,
 			url: file.uri,
-			thumbnailUrl: null,
-			webpublicUrl: null
+			thumbnailUrl: file.uri,
+			webpublicUrl: file.uri
 		});
 	} else {
 		DriveFiles.delete(file.id);

src/tools/clean-remote-files.ts

@@ -1,26 +0,0 @@
-import * as promiseLimit from 'promise-limit';
-import del from '../services/drive/delete-file';
-import { DriveFiles } from '../models';
-import { Not, IsNull } from 'typeorm';
-import { DriveFile } from '../models/entities/drive-file';
-import { ensure } from '../prelude/ensure';
-
-const limit = promiseLimit(16);
-
-DriveFiles.find({
-	userHost: Not(IsNull())
-}).then(async files => {
-	console.log(`there is ${files.length} files`);
-
-	await Promise.all(files.map(file => limit(() => job(file))));
-
-	console.log('ALL DONE');
-});
-
-async function job(file: DriveFile): Promise<any> {
-	file = await DriveFiles.findOne(file.id).then(ensure);
-
-	await del(file, true);
-
-	console.log('done', file.id);
-}