mirror of
https://activitypub.software/TransFem-org/Sharkey.git
synced 2024-12-15 09:15:44 +01:00
978a9bbb3b
* Implement? HttpFetchService
* ✌️
* remove node-fetch
* fix
* refactor
* fix
* gateway timeout
* UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない)
* fix
* add logger and fix url preview
* fix ip check
* enhance logger and error handling
* fix
* fix
* clean up
* Use custom fetcher for ApRequest / ApResolver
* bypassProxyはproxyBypassHostsに判断を委譲するように
* set maxRedirections (default 3, ApRequest/ApResolver: 0)
* fix comment
* handle error s3 upload
* add debug message
* no return await
* Revert "no return await"
This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2.
* reduce maxSockets
* apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger
* Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger"
This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4.
* add logger
* fix
* change logger name
* safe
* デフォルトでUser-Agentを設定
103 lines
2.8 KiB
TypeScript
103 lines
2.8 KiB
TypeScript
import * as fs from 'node:fs';
|
|
import * as stream from 'node:stream';
|
|
import * as util from 'node:util';
|
|
import { Inject, Injectable } from '@nestjs/common';
|
|
import IPCIDR from 'ip-cidr';
|
|
import PrivateIp from 'private-ip';
|
|
import got, * as Got from 'got';
|
|
import chalk from 'chalk';
|
|
import { DI } from '@/di-symbols.js';
|
|
import type { Config } from '@/config.js';
|
|
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
|
|
import { createTemp } from '@/misc/create-temp.js';
|
|
import { StatusError } from '@/misc/status-error.js';
|
|
import { LoggerService } from '@/core/LoggerService.js';
|
|
import type Logger from '@/logger.js';
|
|
import { buildConnector } from 'undici';
|
|
|
|
const pipeline = util.promisify(stream.pipeline);
|
|
import { bindThis } from '@/decorators.js';
|
|
|
|
@Injectable()
|
|
export class DownloadService {
|
|
private logger: Logger;
|
|
private undiciFetcher: UndiciFetcher;
|
|
|
|
constructor(
|
|
@Inject(DI.config)
|
|
private config: Config,
|
|
|
|
private httpRequestService: HttpRequestService,
|
|
private loggerService: LoggerService,
|
|
) {
|
|
this.logger = this.loggerService.getLogger('download');
|
|
|
|
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
|
|
{
|
|
connect: process.env.NODE_ENV === 'development' ?
|
|
this.httpRequestService.clientDefaults.connect
|
|
:
|
|
this.httpRequestService.getConnectorWithIpCheck(
|
|
buildConnector({
|
|
...this.httpRequestService.clientDefaults.connect,
|
|
}),
|
|
(ip) => !this.isPrivateIp(ip)
|
|
),
|
|
bodyTimeout: 30 * 1000,
|
|
},
|
|
{
|
|
connect: this.httpRequestService.clientDefaults.connect,
|
|
}
|
|
), this.logger);
|
|
}
|
|
|
|
@bindThis
|
|
public async downloadUrl(url: string, path: string): Promise<void> {
|
|
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
|
|
|
|
const timeout = 30 * 1000;
|
|
const operationTimeout = 60 * 1000;
|
|
const maxSize = this.config.maxFileSize ?? 262144000;
|
|
|
|
const response = await this.undiciFetcher.fetch(url);
|
|
|
|
if (response.body === null) {
|
|
throw new StatusError('No body', 400, 'No body');
|
|
}
|
|
|
|
await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
|
|
|
|
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
|
}
|
|
|
|
@bindThis
|
|
public async downloadTextFile(url: string): Promise<string> {
|
|
// Create temp file
|
|
const [path, cleanup] = await createTemp();
|
|
|
|
this.logger.info(`text file: Temp file is ${path}`);
|
|
|
|
try {
|
|
// write content at URL to temp file
|
|
await this.downloadUrl(url, path);
|
|
|
|
const text = await util.promisify(fs.readFile)(path, 'utf8');
|
|
|
|
return text;
|
|
} finally {
|
|
cleanup();
|
|
}
|
|
}
|
|
|
|
@bindThis
|
|
private isPrivateIp(ip: string): boolean {
|
|
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
|
const cidr = new IPCIDR(net);
|
|
if (cidr.contains(ip)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return PrivateIp(ip) ?? false;
|
|
}
|
|
}
|