diff --git a/.config/example.yml b/.config/example.yml index f75224bf77..7afa56fbe4 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -142,4 +142,4 @@ autoAdmin: true #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 # Media Proxy -#mediaProxy: http://127.0.0.1:3000 +#mediaProxy: https://example.com/proxy diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index b587812352..ae66985627 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1410,7 +1410,9 @@ admin/views/instance.vue: object-storage-s3-info-here: "こちら" object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。" cache-remote-files: "リモートのファイルをキャッシュする" - cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにしておくことをおすすめします。" + cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにするか次のリモートファイルのプロキシを有効にすることをおすすめします。" + proxy-remote-files: "リモートのファイルをプロキシする" + proxy-remote-files-desc: "この設定を有効にすると、未保存または保存容量超過で削除されたリモートファイルをローカルでプロキシし、サムネイルも生成するようになります。" local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量" remote-drive-capacity-mb: "リモートユーザーひとりあたりのドライブ容量" mb: "メガバイト単位" diff --git a/migration/1576869585998-ProxyRemoteFiles.ts b/migration/1576869585998-ProxyRemoteFiles.ts new file mode 100644 index 0000000000..1d15370bb6 --- /dev/null +++ b/migration/1576869585998-ProxyRemoteFiles.ts @@ -0,0 +1,14 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class ProxyRemoteFiles1576869585998 implements MigrationInterface { + name = 'ProxyRemoteFiles1576869585998' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT false`, undefined); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "proxyRemoteFiles"`, undefined); + } + +} diff --git a/src/client/app/admin/views/instance.vue b/src/client/app/admin/views/instance.vue index 223b3531e8..ebc554f955 100644 --- a/src/client/app/admin/views/instance.vue +++ b/src/client/app/admin/views/instance.vue @@ -81,6 +81,7 @@
{{ $t('cache-remote-files') }} + {{ $t('proxy-remote-files') }}
{{ $t('local-drive-capacity-mb') }} @@ -275,6 +276,7 @@ export default Vue.extend({ description: null, languages: null, cacheRemoteFiles: false, + proxyRemoteFiles: false, localDriveCapacityMb: null, remoteDriveCapacityMb: null, maxNoteTextLength: null, @@ -339,6 +341,7 @@ export default Vue.extend({ this.description = meta.description; this.languages = meta.langs.join(' '); this.cacheRemoteFiles = meta.cacheRemoteFiles; + this.proxyRemoteFiles = meta.proxyRemoteFiles; this.localDriveCapacityMb = meta.driveCapacityPerLocalUserMb; this.remoteDriveCapacityMb = meta.driveCapacityPerRemoteUserMb; this.maxNoteTextLength = meta.maxNoteTextLength; @@ -463,6 +466,7 @@ export default Vue.extend({ description: this.description, langs: this.languages ? this.languages.split(' ') : [], cacheRemoteFiles: this.cacheRemoteFiles, + proxyRemoteFiles: this.proxyRemoteFiles, localDriveCapacityMb: parseInt(this.localDriveCapacityMb, 10), remoteDriveCapacityMb: parseInt(this.remoteDriveCapacityMb, 10), maxNoteTextLength: parseInt(this.maxNoteTextLength, 10), diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts index fdd2818238..e5b189ef83 100644 --- a/src/models/entities/meta.ts +++ b/src/models/entities/meta.ts @@ -115,6 +115,11 @@ export class Meta { }) public cacheRemoteFiles: boolean; + @Column('boolean', { + default: false, + }) + public proxyRemoteFiles: boolean; + @Column('varchar', { length: 128, nullable: true diff --git a/src/models/repositories/drive-file.ts b/src/models/repositories/drive-file.ts index a20d393044..5e3c1e94b8 100644 --- a/src/models/repositories/drive-file.ts +++ b/src/models/repositories/drive-file.ts @@ -7,6 +7,9 @@ import { ensure } from '../../prelude/ensure'; import { awaitAll } from '../../prelude/await-all'; import { SchemaType } from '../../misc/schema'; import config from '../../config'; +import { query, appendQuery } from '../../prelude/url'; +import { Meta } from '../entities/meta'; +import { fetchMeta } from '../../misc/fetch-meta'; export type PackedDriveFile = SchemaType; @@ -22,12 +25,39 @@ export class DriveFileRepository extends Repository { ); } - public getPublicUrl(file: DriveFile, thumbnail = false): string | null { - let url = thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url); - if (file.src !== null && file.userHost !== null && config.mediaProxy !== null) { - url = `${config.mediaProxy}/${thumbnail ? 'thumbnail' : ''}?url=${file.src}`; + public getPublicUrl(file: DriveFile, thumbnail = false, meta?: Meta): string | null { + // リモートかつメディアプロキシ + if (file.uri != null && file.userHost != null && config.mediaProxy != null) { + return appendQuery(config.mediaProxy, query({ + url: file.uri, + thumbnail: thumbnail ? '1' : undefined + })); } - return url; + + // リモートかつ期限切れはローカルプロキシを試みる + if (file.uri != null && file.isLink && meta && meta.proxyRemoteFiles) { + const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey; + + if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外 + let ext = ''; + + if (file.name) { + [ext] = (file.name.match(/\.(\w+)$/) || ['']); + } + + if (ext === '') { + if (file.type === 'image/jpeg') ext = '.jpg'; + if (file.type === 'image/png') ext = '.png'; + if (file.type === 'image/webp') ext = '.webp'; + if (file.type === 'image/apng') ext = '.apng'; + if (file.type === 'image/vnd.mozilla.apng') ext = '.apng'; + } + + return `/files/${key}/${key}${ext}`; + } + } + + return thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url); } public async clacDriveUsageOf(user: User['id'] | User): Promise { @@ -87,6 +117,8 @@ export class DriveFileRepository extends Repository { const file = typeof src === 'object' ? src : await this.findOne(src).then(ensure); + const meta = await fetchMeta(); + return await awaitAll({ id: file.id, createdAt: file.createdAt.toISOString(), @@ -96,8 +128,8 @@ export class DriveFileRepository extends Repository { size: file.size, isSensitive: file.isSensitive, properties: file.properties, - url: opts.self ? file.url : this.getPublicUrl(file, false), - thumbnailUrl: this.getPublicUrl(file, true), + url: opts.self ? file.url : this.getPublicUrl(file, false, meta), + thumbnailUrl: this.getPublicUrl(file, true, meta), folderId: file.folderId, folder: opts.detail && file.folderId ? DriveFolders.pack(file.folderId, { detail: true diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index 05a1e25c01..bc37228d0a 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -151,6 +151,13 @@ export const meta = { } }, + proxyRemoteFiles: { + validator: $.optional.bool, + desc: { + 'ja-JP': 'ローカルにないリモートのファイルをプロキシするか否か' + } + }, + enableRecaptcha: { validator: $.optional.bool, desc: { @@ -478,6 +485,10 @@ export default define(meta, async (ps, me) => { set.cacheRemoteFiles = ps.cacheRemoteFiles; } + if (ps.proxyRemoteFiles !== undefined) { + set.proxyRemoteFiles = ps.proxyRemoteFiles; + } + if (ps.enableRecaptcha !== undefined) { set.enableRecaptcha = ps.enableRecaptcha; } diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts index 6df6362a6f..b71c35946e 100644 --- a/src/server/api/endpoints/meta.ts +++ b/src/server/api/endpoints/meta.ts @@ -143,6 +143,7 @@ export default define(meta, async (ps, me) => { driveCapacityPerLocalUserMb: instance.localDriveCapacityMb, driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb, cacheRemoteFiles: instance.cacheRemoteFiles, + proxyRemoteFiles: instance.proxyRemoteFiles, enableRecaptcha: instance.enableRecaptcha, recaptchaSiteKey: instance.recaptchaSiteKey, swPublickey: instance.swPublicKey, diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index a05477488b..e0aea5b42f 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -1,10 +1,16 @@ import * as Koa from 'koa'; import * as send from 'koa-send'; import * as rename from 'rename'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; import { serverLogger } from '..'; import { contentDisposition } from '../../misc/content-disposition'; import { DriveFiles } from '../../models'; import { InternalStorage } from '../../services/drive/internal-storage'; +import { downloadUrl } from '../../misc/donwload-url'; +import { detectMine } from '../../misc/detect-mine'; +import { convertToJpeg, convertToPng, convertToGif, convertToApng } from '../../services/drive/image-processor'; +import { GenerateVideoThumbnail } from '../../services/drive/generate-video-thumbnail'; const assets = `${__dirname}/../../server/file/assets/`; @@ -31,15 +37,70 @@ export default async function(ctx: Koa.Context) { return; } + const isThumbnail = file.thumbnailAccessKey === key; + const isWebpublic = file.webpublicAccessKey === key; + if (!file.storedInternal) { + if (file.isLink && file.uri) { // 期限切れリモートファイル + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + try { + await downloadUrl(file.uri, path); + + const [type, ext] = await detectMine(path); + + const convertFile = async () => { + if (isThumbnail) { + if (['image/jpeg', 'image/webp'].includes(type)) { + return await convertToJpeg(path, 498, 280); + } else if (['image/png'].includes(type)) { + return await convertToPng(path, 498, 280); + } else if (['image/gif'].includes(type)) { + return await convertToGif(path); + } else if (['image/apng', 'image/vnd.mozilla.apng'].includes(type)) { + return await convertToApng(path); + } else if (type.startsWith('video/')) { + return await GenerateVideoThumbnail(path); + } + } + + return { + data: fs.readFileSync(path), + ext, + type, + }; + }; + + const image = await convertFile(); + ctx.body = image.data; + ctx.set('Content-Type', file.type); + ctx.set('Cache-Control', 'max-age=31536000, immutable'); + } catch (e) { + serverLogger.error(e); + + if (typeof e == 'number' && e >= 400 && e < 500) { + ctx.status = e; + ctx.set('Cache-Control', 'max-age=86400'); + } else { + ctx.status = 500; + ctx.set('Cache-Control', 'max-age=300'); + } + } finally { + cleanup(); + } + return; + } + ctx.status = 204; ctx.set('Cache-Control', 'max-age=86400'); return; } - const isThumbnail = file.thumbnailAccessKey === key; - const isWebpublic = file.webpublicAccessKey === key; - if (isThumbnail) { ctx.body = InternalStorage.read(key); ctx.set('Content-Type', 'image/jpeg'); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index b69fef2afe..877075b8b0 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -424,6 +424,10 @@ export default async function( file.url = url; file.thumbnailUrl = url; file.webpublicUrl = url; + // ローカルプロキシ用 + file.accessKey = uuid(); + file.thumbnailAccessKey = 'thumbnail-' + uuid(); + file.webpublicAccessKey = 'webpublic-' + uuid(); } } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 6b17bc313c..18603617d2 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -5,6 +5,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '../chart'; import { createDeleteObjectStorageFileJob } from '../../queue'; import { fetchMeta } from '../../misc/fetch-meta'; import { getS3 } from './s3'; +import { v4 as uuid } from 'uuid'; export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -71,6 +72,10 @@ function postProcess(file: DriveFile, isExpired = false) { thumbnailUrl: file.uri, webpublicUrl: file.uri, size: 0, + // ローカルプロキシ用 + accessKey: uuid(), + thumbnailAccessKey: 'thumbnail-' + uuid(), + webpublicAccessKey: 'webpublic-' + uuid(), }); } else { DriveFiles.delete(file.id);