separate SkRateLimiterService from RateLimiterService and update all usages

This commit is contained in:
Hazelnoot 2024-12-07 13:13:19 -05:00
parent 29c3beaa62
commit f6b256620b
10 changed files with 101 additions and 123 deletions

View file

@ -0,0 +1,21 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { FastifyReply } from 'fastify';
import { LimitInfo } from '@/server/api/SkRateLimiterService.js';
export function sendRateLimitHeaders(reply: FastifyReply, info: LimitInfo): void {
// Number of seconds until the limit has fully reset.
reply.header('X-RateLimit-Clear', info.fullResetSec.toString());
// Number of calls that can be made before being limited.
reply.header('X-RateLimit-Remaining', info.remaining.toString());
if (info.blocked) {
// Number of seconds to wait before trying again. Left for backwards compatibility.
reply.header('Retry-After', info.resetSec.toString());
// Number of milliseconds to wait before trying again.
reply.header('X-RateLimit-Reset', info.resetMs.toString());
}
}

View file

@ -28,13 +28,12 @@ import { bindThis } from '@/decorators.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import { correctFilename } from '@/misc/correct-filename.js';
import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js';
import { RateLimiterService } from '@/server/api/RateLimiterService.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import { AuthenticateService } from '@/server/api/AuthenticateService.js';
import type { IEndpointMeta } from '@/server/api/endpoints.js';
import { RoleService } from '@/core/RoleService.js';
import { RateLimit, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify';
import type Limiter from 'ratelimiter';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -59,7 +58,7 @@ export class FileServerService {
private internalStorageService: InternalStorageService,
private loggerService: LoggerService,
private authenticateService: AuthenticateService,
private rateLimiterService: RateLimiterService,
private rateLimiterService: SkRateLimiterService,
private roleService: RoleService,
) {
this.logger = this.loggerService.getLogger('server', 'gray');
@ -634,42 +633,37 @@ export class FileServerService {
}
private async checkResourceLimit(reply: FastifyReply, actor: string, group: string, resource: string, factor = 1): Promise<boolean> {
const limit = {
const limit: RateLimit = {
// Group by resource
key: `${group}${resource}`,
type: 'bucket',
// Maximum of 10 requests / 10 minutes
max: 10,
duration: 1000 * 60 * 10,
// Maximum of 10 requests, average rate of 1 per minute
size: 10,
dripRate: 1000 * 60,
};
return await this.checkLimit(reply, actor, limit, factor);
}
private async checkSharedLimit(reply: FastifyReply, actor: string, group: string, factor = 1): Promise<boolean> {
const limit = {
const limit: RateLimit = {
key: group,
type: 'bucket',
// Maximum of 3600 requests per hour, which is an average of 1 per second.
max: 3600,
duration: 1000 * 60 * 60,
// Maximum of 3600 requests, average rate of 1 per second.
size: 3600,
};
return await this.checkLimit(reply, actor, limit, factor);
}
private async checkLimit(reply: FastifyReply, actor: string, limit: IEndpointMeta['limit'] & { key: NonNullable<string> }, factor = 1): Promise<boolean> {
try {
await this.rateLimiterService.limit(limit, actor, factor);
return true;
} catch (err) {
// errはLimiter.LimiterInfoであることが期待される
if (hasRateLimitInfo(err)) {
const cooldownInSeconds = Math.ceil((err.info.resetMs - Date.now()) / 1000);
// もしかするとマイナスになる可能性がなくはないのでマイナスだったら0にしておく
reply.header('Retry-After', Math.max(cooldownInSeconds, 0).toString(10));
}
private async checkLimit(reply: FastifyReply, actor: string, limit: RateLimit, factor = 1): Promise<boolean> {
const info = await this.rateLimiterService.limit(limit, actor, factor);
sendRateLimitHeaders(reply, info);
if (info.blocked) {
reply.code(429);
reply.send({
message: 'Rate limit exceeded. Please try again later.',
@ -679,9 +673,8 @@ export class FileServerService {
return false;
}
return true;
}
}
function hasRateLimitInfo(err: unknown): err is { info: Limiter.LimiterInfo } {
return err != null && typeof(err) === 'object' && 'info' in err;
}

View file

@ -74,10 +74,9 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j
ApiLoggerService,
ApiServerService,
AuthenticateService,
{
provide: RateLimiterService,
useClass: SkRateLimiterService,
},
SkRateLimiterService,
// No longer used, but kept for backwards compatibility
RateLimiterService,
SigninApiService,
SigninWithPasskeyApiService,
SigninService,

View file

@ -8,7 +8,6 @@ import * as fs from 'node:fs';
import * as stream from 'node:stream/promises';
import { Inject, Injectable } from '@nestjs/common';
import * as Sentry from '@sentry/node';
import { LimiterInfo } from 'ratelimiter';
import { DI } from '@/di-symbols.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
@ -19,9 +18,9 @@ import { createTemp } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import type { Config } from '@/config.js';
import { isLimitInfo } from '@/server/api/SkRateLimiterService.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { LegacyRateLimit, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { ApiError } from './error.js';
import { RateLimiterService } from './RateLimiterService.js';
import { ApiLoggerService } from './ApiLoggerService.js';
import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
import type { FastifyRequest, FastifyReply } from 'fastify';
@ -51,7 +50,7 @@ export class ApiCallService implements OnApplicationShutdown {
private userIpsRepository: UserIpsRepository,
private authenticateService: AuthenticateService,
private rateLimiterService: RateLimiterService,
private rateLimiterService: SkRateLimiterService,
private roleService: RoleService,
private apiLoggerService: ApiLoggerService,
) {
@ -67,21 +66,6 @@ export class ApiCallService implements OnApplicationShutdown {
let statusCode = err.httpStatusCode;
if (err.httpStatusCode === 401) {
reply.header('WWW-Authenticate', 'Bearer realm="Misskey"');
} else if (err.code === 'RATE_LIMIT_EXCEEDED') {
const info: unknown = err.info;
const unixEpochInSeconds = Date.now();
if (isLimitInfo(info)) {
// Number of seconds to wait before trying again. Left for backwards compatibility.
reply.header('Retry-After', info.resetSec.toString());
// Number of milliseconds to wait before trying again.
reply.header('X-RateLimit-Reset', info.resetMs.toString());
} else if (typeof(info) === 'object' && info && 'resetMs' in info && typeof(info.resetMs) === 'number') {
const cooldownInSeconds = Math.ceil((info.resetMs - unixEpochInSeconds) / 1000);
// もしかするとマイナスになる可能性がなくはないのでマイナスだったら0にしておく
reply.header('Retry-After', Math.max(cooldownInSeconds, 0).toString(10));
} else {
this.logger.warn(`rate limit information has unexpected type: ${JSON.stringify(info)}`);
}
} else if (err.kind === 'client') {
reply.header('WWW-Authenticate', `Bearer realm="Misskey", error="invalid_request", error_description="${err.message}"`);
statusCode = statusCode ?? 400;
@ -347,40 +331,17 @@ export class ApiCallService implements OnApplicationShutdown {
if (factor > 0) {
// Rate limit
const info = await this.rateLimiterService.limit(limit as IEndpointMeta['limit'] & { key: NonNullable<string> }, limitActor, factor)
.then(info => {
// We always want these headers, because clients need them for pacing.
// Conditional check in case we somehow revert to the old limiter, which does not return info.
if (info) {
// Number of seconds until the limit has fully reset.
reply.header('X-RateLimit-Clear', info.fullResetSec.toString());
// Number of calls that can be made before being limited.
reply.header('X-RateLimit-Remaining', info.remaining.toString());
const info = await this.rateLimiterService.limit(limit as LegacyRateLimit, limitActor, factor);
// Only forward the info object if it's blocked, otherwise we'll reject *all* requests
if (info.blocked) {
return info;
}
}
sendRateLimitHeaders(reply, info);
return undefined;
})
.catch(err => {
// The old limiter throws info instead of returning it.
if ('info' in err) {
return err.info as LimiterInfo;
} else {
throw err;
}
});
if (info) {
if (info.blocked) {
throw new ApiError({
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
httpStatusCode: 429,
}, info);
});
}
}
}

View file

@ -14,6 +14,7 @@ import type { LimitInfo } from '@/server/api/SkRateLimiterService.js';
import { EnvService } from '@/core/EnvService.js';
import type { IEndpointMeta } from './endpoints.js';
/** @deprecated Use SkRateLimiterService instead */
@Injectable()
export class RateLimiterService {
protected readonly logger: Logger;

View file

@ -21,12 +21,13 @@ import { IdService } from '@/core/IdService.js';
import { bindThis } from '@/decorators.js';
import { WebAuthnService } from '@/core/WebAuthnService.js';
import { UserAuthService } from '@/core/UserAuthService.js';
import { RateLimiterService } from './RateLimiterService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import type { MiMeta } from '@/models/_.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { SigninService } from './SigninService.js';
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { isSystemAccount } from '@/misc/is-system-account.js';
import type { MiMeta } from '@/models/_.js';
@Injectable()
export class SigninApiService {
@ -47,7 +48,7 @@ export class SigninApiService {
private signinsRepository: SigninsRepository,
private idService: IdService,
private rateLimiterService: RateLimiterService,
private rateLimiterService: SkRateLimiterService,
private signinService: SigninService,
private userAuthService: UserAuthService,
private webAuthnService: WebAuthnService,
@ -79,10 +80,12 @@ export class SigninApiService {
return { error };
}
try {
// not more than 1 attempt per second and not more than 10 attempts per hour
await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
} catch (err) {
const rateLimit = await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
sendRateLimitHeaders(reply, rateLimit);
if (rateLimit.blocked) {
reply.code(429);
return {
error: {

View file

@ -21,10 +21,11 @@ import { WebAuthnService } from '@/core/WebAuthnService.js';
import Logger from '@/logger.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IdentifiableError } from '@/misc/identifiable-error.js';
import { RateLimiterService } from './RateLimiterService.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { SigninService } from './SigninService.js';
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
@Injectable()
export class SigninWithPasskeyApiService {
@ -43,7 +44,7 @@ export class SigninWithPasskeyApiService {
private signinsRepository: SigninsRepository,
private idService: IdService,
private rateLimiterService: RateLimiterService,
private rateLimiterService: SkRateLimiterService,
private signinService: SigninService,
private webAuthnService: WebAuthnService,
private loggerService: LoggerService,
@ -84,11 +85,13 @@ export class SigninWithPasskeyApiService {
return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' });
};
try {
// Not more than 1 API call per 250ms and not more than 100 attempts per 30min
// NOTE: 1 Sign-in require 2 API calls
await this.rateLimiterService.limit({ key: 'signin-with-passkey', duration: 60 * 30 * 1000, max: 200, minInterval: 250 }, getIpHash(request.ip));
} catch (err) {
// Not more than 1 API call per 250ms and not more than 100 attempts per 30min
// NOTE: 1 Sign-in require 2 API calls
const rateLimit = await this.rateLimiterService.limit({ key: 'signin-with-passkey', duration: 60 * 30 * 1000, max: 200, minInterval: 250 }, getIpHash(request.ip));
sendRateLimitHeaders(reply, rateLimit);
if (rateLimit.blocked) {
reply.code(429);
return {
error: {

View file

@ -10,7 +10,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import { TimeService } from '@/core/TimeService.js';
import { EnvService } from '@/core/EnvService.js';
import { DI } from '@/di-symbols.js';
import { RateLimiterService } from './RateLimiterService.js';
import type Logger from '@/logger.js';
/**
* Metadata about the current status of a rate limiter
@ -51,18 +51,6 @@ export interface LimitInfo {
fullResetMs: number;
}
export function isLimitInfo(info: unknown): info is LimitInfo {
if (info == null) return false;
if (typeof(info) !== 'object') return false;
if (!('blocked' in info) || typeof(info.blocked) !== 'boolean') return false;
if (!('remaining' in info) || typeof(info.remaining) !== 'number') return false;
if (!('resetSec' in info) || typeof(info.resetSec) !== 'number') return false;
if (!('resetMs' in info) || typeof(info.resetMs) !== 'number') return false;
if (!('fullResetSec' in info) || typeof(info.fullResetSec) !== 'number') return false;
if (!('fullResetMs' in info) || typeof(info.fullResetMs) !== 'number') return false;
return true;
}
/**
* Rate limit based on "leaky bucket" logic.
* The bucket count increases with each call, and decreases gradually at a given rate.
@ -99,10 +87,10 @@ export interface RateLimit {
}
export type SupportedRateLimit = RateLimit | LegacyRateLimit;
export type LegacyRateLimit = IEndpointMeta['limit'] & { key: NonNullable<string>, type: undefined | 'legacy' };
export type LegacyRateLimit = IEndpointMeta['limit'] & { key: NonNullable<string>, type?: undefined };
export function isLegacyRateLimit(limit: SupportedRateLimit): limit is LegacyRateLimit {
return limit.type === undefined || limit.type === 'legacy';
return limit.type === undefined;
}
export function hasMinLimit(limit: LegacyRateLimit): limit is LegacyRateLimit & { minInterval: number } {
@ -110,13 +98,16 @@ export function hasMinLimit(limit: LegacyRateLimit): limit is LegacyRateLimit &
}
@Injectable()
export class SkRateLimiterService extends RateLimiterService {
export class SkRateLimiterService {
private readonly logger: Logger;
private readonly disabled: boolean;
constructor(
@Inject(TimeService)
private readonly timeService: TimeService,
@Inject(DI.redis)
redisClient: Redis.Redis,
private readonly redisClient: Redis.Redis,
@Inject(LoggerService)
loggerService: LoggerService,
@ -124,7 +115,8 @@ export class SkRateLimiterService extends RateLimiterService {
@Inject(EnvService)
envService: EnvService,
) {
super(redisClient, loggerService, envService);
this.logger = loggerService.getLogger('limiter');
this.disabled = envService.env.NODE_ENV !== 'production';
}
public async limit(limit: SupportedRateLimit, actor: string, factor = 1): Promise<LimitInfo> {

View file

@ -7,6 +7,8 @@ import { EventEmitter } from 'events';
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import * as WebSocket from 'ws';
import proxyAddr from 'proxy-addr';
import ms from 'ms';
import { DI } from '@/di-symbols.js';
import type { UsersRepository, MiAccessToken } from '@/models/_.js';
import { NoteReadService } from '@/core/NoteReadService.js';
@ -16,18 +18,15 @@ import { CacheService } from '@/core/CacheService.js';
import { MiLocalUser } from '@/models/User.js';
import { UserService } from '@/core/UserService.js';
import { ChannelFollowingService } from '@/core/ChannelFollowingService.js';
import { RoleService } from '@/core/RoleService.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import { LoggerService } from '@/core/LoggerService.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
import MainStreamConnection from './stream/Connection.js';
import { ChannelsService } from './stream/ChannelsService.js';
import { RateLimiterService } from './RateLimiterService.js';
import { RoleService } from '@/core/RoleService.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import proxyAddr from 'proxy-addr';
import ms from 'ms';
import type * as http from 'node:http';
import type { IEndpointMeta } from './endpoints.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
@Injectable()
export class StreamingApiServerService {
@ -49,7 +48,7 @@ export class StreamingApiServerService {
private notificationService: NotificationService,
private usersService: UserService,
private channelFollowingService: ChannelFollowingService,
private rateLimiterService: RateLimiterService,
private rateLimiterService: SkRateLimiterService,
private roleService: RoleService,
private loggerService: LoggerService,
) {
@ -73,9 +72,8 @@ export class StreamingApiServerService {
if (factor <= 0) return false;
// Rate limit
return await this.rateLimiterService.limit(limit, limitActor, factor)
.then(() => { return false; })
.catch(err => { return true; });
const rateLimit = await this.rateLimiterService.limit(limit, limitActor, factor);
return rateLimit.blocked;
}
@bindThis

View file

@ -17,16 +17,23 @@ import { GlobalModule } from '@/GlobalModule.js';
import { DI } from '@/di-symbols.js';
import { CoreModule } from '@/core/CoreModule.js';
import { SigninWithPasskeyApiService } from '@/server/api/SigninWithPasskeyApiService.js';
import { RateLimiterService } from '@/server/api/RateLimiterService.js';
import { WebAuthnService } from '@/core/WebAuthnService.js';
import { SigninService } from '@/server/api/SigninService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { LimitInfo, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
const moduleMocker = new ModuleMocker(global);
class FakeLimiter {
public async limit() {
return;
public async limit(): Promise<LimitInfo> {
return {
blocked: false,
remaining: Number.MAX_SAFE_INTEGER,
resetMs: 0,
resetSec: 0,
fullResetMs: 0,
fullResetSec: 0,
};
}
}
@ -90,7 +97,7 @@ describe('SigninWithPasskeyApiService', () => {
imports: [GlobalModule, CoreModule],
providers: [
SigninWithPasskeyApiService,
{ provide: RateLimiterService, useClass: FakeLimiter },
{ provide: SkRateLimiterService, useClass: FakeLimiter },
{ provide: SigninService, useClass: FakeSigninService },
],
}).useMocker((token) => {