Sharkey/packages/backend/src/queue/QueueProcessorService.ts

357 lines
17 KiB
TypeScript
Raw Normal View History

2023-05-29 06:30:26 +02:00
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Bull from 'bullmq';
2022-09-20 22:33:11 +02:00
import type { Config } from '@/config.js';
2022-09-17 20:27:08 +02:00
import { DI } from '@/di-symbols.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
2022-09-17 20:27:08 +02:00
import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js';
import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js';
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
import { InboxProcessorService } from './processors/InboxProcessorService.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js';
import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js';
import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js';
import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js';
import { ImportAntennasProcessorService } from './processors/ImportAntennasProcessorService.js';
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js';
import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesProcessorService.js';
import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
import { RelationshipProcessorService } from './processors/RelationshipProcessorService.js';
import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js';
import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js';
import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js';
import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js';
import { CleanProcessorService } from './processors/CleanProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
2022-09-17 20:27:08 +02:00
import { QueueLoggerService } from './QueueLoggerService.js';
import { QUEUE, baseQueueOptions } from './const.js';
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
function httpRelatedBackoff(attemptsMade: number) {
const baseDelay = 60 * 1000; // 1min
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
backoff = Math.min(backoff, maxBackoff);
backoff += Math.round(backoff * Math.random() * 0.2);
return backoff;
}
function getJobInfo(job: Bull.Job | undefined, increment = false): string {
if (job == null) return '-';
const age = Date.now() - job.timestamp;
const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m`
: age > 10000 ? `${Math.floor(age / 1000)}s`
: `${age}ms`;
// onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする
const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
const maxAttempts = job.opts ? job.opts.attempts : 0;
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
}
2022-09-17 20:27:08 +02:00
@Injectable()
2023-05-29 06:30:26 +02:00
export class QueueProcessorService implements OnApplicationShutdown {
2022-09-18 20:11:50 +02:00
private logger: Logger;
2023-05-29 06:30:26 +02:00
private systemQueueWorker: Bull.Worker;
private dbQueueWorker: Bull.Worker;
private deliverQueueWorker: Bull.Worker;
private inboxQueueWorker: Bull.Worker;
private webhookDeliverQueueWorker: Bull.Worker;
private relationshipQueueWorker: Bull.Worker;
private objectStorageQueueWorker: Bull.Worker;
private endedPollNotificationQueueWorker: Bull.Worker;
2022-09-17 20:27:08 +02:00
constructor(
@Inject(DI.config)
private config: Config,
private queueLoggerService: QueueLoggerService,
private webhookDeliverProcessorService: WebhookDeliverProcessorService,
private endedPollNotificationProcessorService: EndedPollNotificationProcessorService,
private deliverProcessorService: DeliverProcessorService,
private inboxProcessorService: InboxProcessorService,
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
private exportFollowingProcessorService: ExportFollowingProcessorService,
private exportMutingProcessorService: ExportMutingProcessorService,
private exportBlockingProcessorService: ExportBlockingProcessorService,
private exportUserListsProcessorService: ExportUserListsProcessorService,
private exportAntennasProcessorService: ExportAntennasProcessorService,
private importFollowingProcessorService: ImportFollowingProcessorService,
private importMutingProcessorService: ImportMutingProcessorService,
private importBlockingProcessorService: ImportBlockingProcessorService,
private importUserListsProcessorService: ImportUserListsProcessorService,
private importCustomEmojisProcessorService: ImportCustomEmojisProcessorService,
private importAntennasProcessorService: ImportAntennasProcessorService,
private deleteAccountProcessorService: DeleteAccountProcessorService,
private deleteFileProcessorService: DeleteFileProcessorService,
private cleanRemoteFilesProcessorService: CleanRemoteFilesProcessorService,
private relationshipProcessorService: RelationshipProcessorService,
private tickChartsProcessorService: TickChartsProcessorService,
private resyncChartsProcessorService: ResyncChartsProcessorService,
private cleanChartsProcessorService: CleanChartsProcessorService,
private aggregateRetentionProcessorService: AggregateRetentionProcessorService,
private checkExpiredMutingsProcessorService: CheckExpiredMutingsProcessorService,
private cleanProcessorService: CleanProcessorService,
2022-09-17 20:27:08 +02:00
) {
2022-09-18 20:11:50 +02:00
this.logger = this.queueLoggerService.logger;
2022-09-17 20:27:08 +02:00
function renderError(e: Error): any {
2022-11-11 08:55:16 +01:00
if (e) { // 何故かeがundefinedで来ることがある
return {
stack: e.stack,
message: e.message,
name: e.name,
};
} else {
return {
stack: '?',
message: '?',
name: '?',
};
}
2022-09-17 20:27:08 +02:00
}
//#region system
2023-05-29 06:30:26 +02:00
this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => {
switch (job.name) {
case 'tickCharts': return this.tickChartsProcessorService.process();
case 'resyncCharts': return this.resyncChartsProcessorService.process();
case 'cleanCharts': return this.cleanChartsProcessorService.process();
case 'aggregateRetention': return this.aggregateRetentionProcessorService.process();
case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process();
case 'clean': return this.cleanProcessorService.process();
default: throw new Error(`unrecognized job type ${job.name} for system`);
}
}, {
...baseQueueOptions(this.config, QUEUE.SYSTEM),
2023-05-29 06:30:26 +02:00
autorun: false,
});
2022-09-18 20:11:50 +02:00
const systemLogger = this.logger.createSubLogger('system');
2023-05-29 06:30:26 +02:00
this.systemQueueWorker
2022-09-17 20:27:08 +02:00
.on('active', (job) => systemLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region db
2023-05-29 06:30:26 +02:00
this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
switch (job.name) {
case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
case 'exportNotes': return this.exportNotesProcessorService.process(job);
case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
case 'exportFollowing': return this.exportFollowingProcessorService.process(job);
case 'exportMuting': return this.exportMutingProcessorService.process(job);
case 'exportBlocking': return this.exportBlockingProcessorService.process(job);
case 'exportUserLists': return this.exportUserListsProcessorService.process(job);
case 'exportAntennas': return this.exportAntennasProcessorService.process(job);
case 'importFollowing': return this.importFollowingProcessorService.process(job);
case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job);
case 'importMuting': return this.importMutingProcessorService.process(job);
case 'importBlocking': return this.importBlockingProcessorService.process(job);
case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job);
case 'importUserLists': return this.importUserListsProcessorService.process(job);
case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job);
case 'importAntennas': return this.importAntennasProcessorService.process(job);
case 'deleteAccount': return this.deleteAccountProcessorService.process(job);
default: throw new Error(`unrecognized job type ${job.name} for db`);
}
}, {
...baseQueueOptions(this.config, QUEUE.DB),
2023-05-29 06:30:26 +02:00
autorun: false,
});
const dbLogger = this.logger.createSubLogger('db');
2023-05-29 06:30:26 +02:00
this.dbQueueWorker
2022-09-17 20:27:08 +02:00
.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region deliver
2023-05-29 06:30:26 +02:00
this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.DELIVER),
2023-05-29 06:30:26 +02:00
autorun: false,
concurrency: this.config.deliverJobConcurrency ?? 128,
limiter: {
max: this.config.deliverJobPerSec ?? 128,
duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
});
const deliverLogger = this.logger.createSubLogger('deliver');
2023-05-29 06:30:26 +02:00
this.deliverQueueWorker
.on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region inbox
2023-05-29 06:30:26 +02:00
this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.INBOX),
2023-05-29 06:30:26 +02:00
autorun: false,
concurrency: this.config.inboxJobConcurrency ?? 16,
limiter: {
max: this.config.inboxJobPerSec ?? 16,
duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
2022-09-17 20:27:08 +02:00
});
const inboxLogger = this.logger.createSubLogger('inbox');
2023-05-29 06:30:26 +02:00
this.inboxQueueWorker
.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region webhook deliver
2023-05-29 06:30:26 +02:00
this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER),
2023-05-29 06:30:26 +02:00
autorun: false,
concurrency: 64,
limiter: {
max: 64,
duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
});
const webhookLogger = this.logger.createSubLogger('webhook');
2023-05-29 06:30:26 +02:00
this.webhookDeliverQueueWorker
.on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region relationship
2023-05-29 06:30:26 +02:00
this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => {
switch (job.name) {
case 'follow': return this.relationshipProcessorService.processFollow(job);
case 'unfollow': return this.relationshipProcessorService.processUnfollow(job);
case 'block': return this.relationshipProcessorService.processBlock(job);
case 'unblock': return this.relationshipProcessorService.processUnblock(job);
default: throw new Error(`unrecognized job type ${job.name} for relationship`);
}
2022-09-17 20:27:08 +02:00
}, {
...baseQueueOptions(this.config, QUEUE.RELATIONSHIP),
2023-05-29 06:30:26 +02:00
autorun: false,
concurrency: this.config.relashionshipJobConcurrency ?? 16,
limiter: {
max: this.config.relashionshipJobPerSec ?? 64,
duration: 1000,
},
2022-09-17 20:27:08 +02:00
});
const relationshipLogger = this.logger.createSubLogger('relationship');
2023-05-29 06:30:26 +02:00
this.relationshipQueueWorker
.on('active', (job) => relationshipLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region object storage
2023-05-29 06:30:26 +02:00
this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => {
switch (job.name) {
case 'deleteFile': return this.deleteFileProcessorService.process(job);
case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job);
default: throw new Error(`unrecognized job type ${job.name} for objectStorage`);
}
2022-09-17 20:27:08 +02:00
}, {
...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE),
2023-05-29 06:30:26 +02:00
autorun: false,
concurrency: 16,
2022-09-17 20:27:08 +02:00
});
const objectStorageLogger = this.logger.createSubLogger('objectStorage');
2023-05-29 06:30:26 +02:00
this.objectStorageQueueWorker
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region ended poll notification
2023-05-29 06:30:26 +02:00
this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),
2023-05-29 06:30:26 +02:00
autorun: false,
});
//#endregion
2022-09-17 20:27:08 +02:00
}
2023-05-29 06:30:26 +02:00
@bindThis
public start() {
this.systemQueueWorker.run();
this.dbQueueWorker.run();
this.deliverQueueWorker.run();
this.inboxQueueWorker.run();
this.webhookDeliverQueueWorker.run();
this.relationshipQueueWorker.run();
this.objectStorageQueueWorker.run();
this.endedPollNotificationQueueWorker.run();
}
@bindThis
public async stop(): Promise<void> {
await Promise.all([
this.systemQueueWorker.close(),
this.dbQueueWorker.close(),
this.deliverQueueWorker.close(),
this.inboxQueueWorker.close(),
this.webhookDeliverQueueWorker.close(),
this.relationshipQueueWorker.close(),
this.objectStorageQueueWorker.close(),
this.endedPollNotificationQueueWorker.close(),
]);
}
@bindThis
public async onApplicationShutdown(signal?: string | undefined): Promise<void> {
await this.stop();
}
2022-09-17 20:27:08 +02:00
}