From 2f09d69773dcc6c4607b2c9e1f5c86cc1337ece1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:29:03 +0900 Subject: [PATCH] =?UTF-8?q?fix(backend):=20=E3=82=AD=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=AD=E3=82=B0=E3=82=92?= =?UTF-8?q?=E7=B0=A1=E7=95=A5=E5=8C=96=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=20(#14748)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * reduce federation log spam * Don't record stack trace for unrecoverable errors. * Avoid logging duplicate stace traces. (cherry picked from commit ed0570110bf8cb8e8959591dccfa3c35999106ce) * improve error summaries (cherry picked from commit 20dd66f735d9778df0371001e303549dce619260) * fix lint errors (cherry picked from commit 83869e1c470b12b3bf4b23d885514d926620662a) * condense job info (cherry picked from commit 786702e076ad1af14538849512ad31c0ced7afe6) * fix maxAttempts calculation (cherry picked from commit b4d10aa8f821e594ec9c907eb2a5bdb3c73c67d5) * condense error info (cherry picked from commit f62cd8941ced74a4865aa5eae4f4a1c7aa1d30f1) * normalize ID logging (cherry picked from commit d8e1e4890d28347239162e26235eb68b1ff96654) * further condense error details (cherry picked from commit d867c2089b3b24680df0713a2aa0914789e45670) * collapse AbortErrors (cherry picked from commit 5171ba7113ebc7242527768afb9ab4cec534e3b3) * don't log job name unless it has one (cherry picked from commit a5316c06ed770b60f7b4c7ff5aa8c71cc0558db7) * Update Changelog * Record origin --------- Co-authored-by: Hazel K --- CHANGELOG.md | 4 + .../src/queue/QueueProcessorService.ts | 86 +++++++++++-------- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 030dbfda28..143b63f7b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ - Feat: モデレータ権限を持つユーザが全員7日間活動しなかった場合は自動的に招待制へと移行するように ( #13437 ) - Fix: `admin/emoji/update`エンドポイントのidのみ指定した時不正なエラーが発生するバグを修正 +### Server +- Fix: キューのエラーログを簡略化するように + (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/649) + ## 2024.10.0 ### Note diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 85e148e900..6940e1c188 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -67,7 +67,7 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string { // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする const currentAttempts = job.attemptsMade + (increment ? 1 : 0); - const maxAttempts = job.opts ? job.opts.attempts : 0; + const maxAttempts = job.opts.attempts ?? 0; return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; } @@ -126,20 +126,30 @@ export class QueueProcessorService implements OnApplicationShutdown { ) { this.logger = this.queueLoggerService.logger; - function renderError(e: Error): any { - if (e) { // 何故かeがundefinedで来ることがある - return { - stack: e.stack, - message: e.message, - name: e.name, - }; - } else { - return { - stack: '?', - message: '?', - name: '?', - }; + function renderError(e?: Error) { + // 何故かeがundefinedで来ることがある + if (!e) return '?'; + + if (e instanceof Bull.UnrecoverableError || e.name === 'AbortError') { + return `${e.name}: ${e.message}`; } + + return { + stack: e.stack, + message: e.message, + name: e.name, + }; + } + + function renderJob(job?: Bull.Job) { + if (!job) return '?'; + + return { + name: job.name || undefined, + info: getJobInfo(job), + failedReason: job.failedReason || undefined, + data: job.data, + }; } //#region system @@ -175,15 +185,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err: Error) => { - logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.message}`, { + Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -232,15 +242,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.message}`, { + Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -272,15 +282,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: Deliver: ${err.message}`, { + Sentry.captureMessage(`Queue: Deliver: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -312,15 +322,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }); + logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job: renderJob(job), e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: Inbox: ${err.message}`, { + Sentry.captureMessage(`Queue: Inbox: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -352,15 +362,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.message}`, { + Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -392,15 +402,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.message}`, { + Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -439,15 +449,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.message}`, { + Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -480,15 +490,15 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.message}`, { + Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', extra: { job, err }, }); } }) - .on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion