diff --git a/.config/example.yml b/.config/example.yml
index f75224bf77..7afa56fbe4 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -142,4 +142,4 @@ autoAdmin: true
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Media Proxy
-#mediaProxy: http://127.0.0.1:3000
+#mediaProxy: https://example.com/proxy
diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index b587812352..ae66985627 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -1410,7 +1410,9 @@ admin/views/instance.vue:
object-storage-s3-info-here: "こちら"
object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。"
cache-remote-files: "リモートのファイルをキャッシュする"
- cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにしておくことをおすすめします。"
+ cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにするか次のリモートファイルのプロキシを有効にすることをおすすめします。"
+ proxy-remote-files: "リモートのファイルをプロキシする"
+ proxy-remote-files-desc: "この設定を有効にすると、未保存または保存容量超過で削除されたリモートファイルをローカルでプロキシし、サムネイルも生成するようになります。"
local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量"
remote-drive-capacity-mb: "リモートユーザーひとりあたりのドライブ容量"
mb: "メガバイト単位"
diff --git a/migration/1576869585998-ProxyRemoteFiles.ts b/migration/1576869585998-ProxyRemoteFiles.ts
new file mode 100644
index 0000000000..1d15370bb6
--- /dev/null
+++ b/migration/1576869585998-ProxyRemoteFiles.ts
@@ -0,0 +1,14 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class ProxyRemoteFiles1576869585998 implements MigrationInterface {
+ name = 'ProxyRemoteFiles1576869585998'
+
+ public async up(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT false`, undefined);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "proxyRemoteFiles"`, undefined);
+ }
+
+}
diff --git a/src/client/app/admin/views/instance.vue b/src/client/app/admin/views/instance.vue
index 223b3531e8..ebc554f955 100644
--- a/src/client/app/admin/views/instance.vue
+++ b/src/client/app/admin/views/instance.vue
@@ -81,6 +81,7 @@
</section>
<section>
<ui-switch v-model="cacheRemoteFiles">{{ $t('cache-remote-files') }}<template #desc>{{ $t('cache-remote-files-desc') }}</template></ui-switch>
+ <ui-switch v-model="proxyRemoteFiles">{{ $t('proxy-remote-files') }}<template #desc>{{ $t('proxy-remote-files-desc') }}</template></ui-switch>
</section>
<section class="fit-top fit-bottom">
<ui-input v-model="localDriveCapacityMb" type="number">{{ $t('local-drive-capacity-mb') }}<template #suffix>MB</template><template #desc>{{ $t('mb') }}</template></ui-input>
@@ -275,6 +276,7 @@ export default Vue.extend({
description: null,
languages: null,
cacheRemoteFiles: false,
+ proxyRemoteFiles: false,
localDriveCapacityMb: null,
remoteDriveCapacityMb: null,
maxNoteTextLength: null,
@@ -339,6 +341,7 @@ export default Vue.extend({
this.description = meta.description;
this.languages = meta.langs.join(' ');
this.cacheRemoteFiles = meta.cacheRemoteFiles;
+ this.proxyRemoteFiles = meta.proxyRemoteFiles;
this.localDriveCapacityMb = meta.driveCapacityPerLocalUserMb;
this.remoteDriveCapacityMb = meta.driveCapacityPerRemoteUserMb;
this.maxNoteTextLength = meta.maxNoteTextLength;
@@ -463,6 +466,7 @@ export default Vue.extend({
description: this.description,
langs: this.languages ? this.languages.split(' ') : [],
cacheRemoteFiles: this.cacheRemoteFiles,
+ proxyRemoteFiles: this.proxyRemoteFiles,
localDriveCapacityMb: parseInt(this.localDriveCapacityMb, 10),
remoteDriveCapacityMb: parseInt(this.remoteDriveCapacityMb, 10),
maxNoteTextLength: parseInt(this.maxNoteTextLength, 10),
diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts
index fdd2818238..e5b189ef83 100644
--- a/src/models/entities/meta.ts
+++ b/src/models/entities/meta.ts
@@ -115,6 +115,11 @@ export class Meta {
})
public cacheRemoteFiles: boolean;
+ @Column('boolean', {
+ default: false,
+ })
+ public proxyRemoteFiles: boolean;
+
@Column('varchar', {
length: 128,
nullable: true
diff --git a/src/models/repositories/drive-file.ts b/src/models/repositories/drive-file.ts
index a20d393044..5e3c1e94b8 100644
--- a/src/models/repositories/drive-file.ts
+++ b/src/models/repositories/drive-file.ts
@@ -7,6 +7,9 @@ import { ensure } from '../../prelude/ensure';
import { awaitAll } from '../../prelude/await-all';
import { SchemaType } from '../../misc/schema';
import config from '../../config';
+import { query, appendQuery } from '../../prelude/url';
+import { Meta } from '../entities/meta';
+import { fetchMeta } from '../../misc/fetch-meta';
export type PackedDriveFile = SchemaType<typeof packedDriveFileSchema>;
@@ -22,12 +25,39 @@ export class DriveFileRepository extends Repository<DriveFile> {
);
}
- public getPublicUrl(file: DriveFile, thumbnail = false): string | null {
- let url = thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url);
- if (file.src !== null && file.userHost !== null && config.mediaProxy !== null) {
- url = `${config.mediaProxy}/${thumbnail ? 'thumbnail' : ''}?url=${file.src}`;
+ public getPublicUrl(file: DriveFile, thumbnail = false, meta?: Meta): string | null {
+ // リモートかつメディアプロキシ
+ if (file.uri != null && file.userHost != null && config.mediaProxy != null) {
+ return appendQuery(config.mediaProxy, query({
+ url: file.uri,
+ thumbnail: thumbnail ? '1' : undefined
+ }));
}
- return url;
+
+ // リモートかつ期限切れはローカルプロキシを試みる
+ if (file.uri != null && file.isLink && meta && meta.proxyRemoteFiles) {
+ const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey;
+
+ if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外
+ let ext = '';
+
+ if (file.name) {
+ [ext] = (file.name.match(/\.(\w+)$/) || ['']);
+ }
+
+ if (ext === '') {
+ if (file.type === 'image/jpeg') ext = '.jpg';
+ if (file.type === 'image/png') ext = '.png';
+ if (file.type === 'image/webp') ext = '.webp';
+ if (file.type === 'image/apng') ext = '.apng';
+ if (file.type === 'image/vnd.mozilla.apng') ext = '.apng';
+ }
+
+ return `/files/${key}/${key}${ext}`;
+ }
+ }
+
+ return thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url);
}
public async clacDriveUsageOf(user: User['id'] | User): Promise<number> {
@@ -87,6 +117,8 @@ export class DriveFileRepository extends Repository<DriveFile> {
const file = typeof src === 'object' ? src : await this.findOne(src).then(ensure);
+ const meta = await fetchMeta();
+
return await awaitAll({
id: file.id,
createdAt: file.createdAt.toISOString(),
@@ -96,8 +128,8 @@ export class DriveFileRepository extends Repository<DriveFile> {
size: file.size,
isSensitive: file.isSensitive,
properties: file.properties,
- url: opts.self ? file.url : this.getPublicUrl(file, false),
- thumbnailUrl: this.getPublicUrl(file, true),
+ url: opts.self ? file.url : this.getPublicUrl(file, false, meta),
+ thumbnailUrl: this.getPublicUrl(file, true, meta),
folderId: file.folderId,
folder: opts.detail && file.folderId ? DriveFolders.pack(file.folderId, {
detail: true
diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts
index 05a1e25c01..bc37228d0a 100644
--- a/src/server/api/endpoints/admin/update-meta.ts
+++ b/src/server/api/endpoints/admin/update-meta.ts
@@ -151,6 +151,13 @@ export const meta = {
}
},
+ proxyRemoteFiles: {
+ validator: $.optional.bool,
+ desc: {
+ 'ja-JP': 'ローカルにないリモートのファイルをプロキシするか否か'
+ }
+ },
+
enableRecaptcha: {
validator: $.optional.bool,
desc: {
@@ -478,6 +485,10 @@ export default define(meta, async (ps, me) => {
set.cacheRemoteFiles = ps.cacheRemoteFiles;
}
+ if (ps.proxyRemoteFiles !== undefined) {
+ set.proxyRemoteFiles = ps.proxyRemoteFiles;
+ }
+
if (ps.enableRecaptcha !== undefined) {
set.enableRecaptcha = ps.enableRecaptcha;
}
diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts
index 6df6362a6f..b71c35946e 100644
--- a/src/server/api/endpoints/meta.ts
+++ b/src/server/api/endpoints/meta.ts
@@ -143,6 +143,7 @@ export default define(meta, async (ps, me) => {
driveCapacityPerLocalUserMb: instance.localDriveCapacityMb,
driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb,
cacheRemoteFiles: instance.cacheRemoteFiles,
+ proxyRemoteFiles: instance.proxyRemoteFiles,
enableRecaptcha: instance.enableRecaptcha,
recaptchaSiteKey: instance.recaptchaSiteKey,
swPublickey: instance.swPublicKey,
diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts
index a05477488b..e0aea5b42f 100644
--- a/src/server/file/send-drive-file.ts
+++ b/src/server/file/send-drive-file.ts
@@ -1,10 +1,16 @@
import * as Koa from 'koa';
import * as send from 'koa-send';
import * as rename from 'rename';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
import { serverLogger } from '..';
import { contentDisposition } from '../../misc/content-disposition';
import { DriveFiles } from '../../models';
import { InternalStorage } from '../../services/drive/internal-storage';
+import { downloadUrl } from '../../misc/donwload-url';
+import { detectMine } from '../../misc/detect-mine';
+import { convertToJpeg, convertToPng, convertToGif, convertToApng } from '../../services/drive/image-processor';
+import { GenerateVideoThumbnail } from '../../services/drive/generate-video-thumbnail';
const assets = `${__dirname}/../../server/file/assets/`;
@@ -31,15 +37,70 @@ export default async function(ctx: Koa.Context) {
return;
}
+ const isThumbnail = file.thumbnailAccessKey === key;
+ const isWebpublic = file.webpublicAccessKey === key;
+
if (!file.storedInternal) {
+ if (file.isLink && file.uri) { // 期限切れリモートファイル
+ const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+ tmp.file((e, path, fd, cleanup) => {
+ if (e) return rej(e);
+ res([path, cleanup]);
+ });
+ });
+
+ try {
+ await downloadUrl(file.uri, path);
+
+ const [type, ext] = await detectMine(path);
+
+ const convertFile = async () => {
+ if (isThumbnail) {
+ if (['image/jpeg', 'image/webp'].includes(type)) {
+ return await convertToJpeg(path, 498, 280);
+ } else if (['image/png'].includes(type)) {
+ return await convertToPng(path, 498, 280);
+ } else if (['image/gif'].includes(type)) {
+ return await convertToGif(path);
+ } else if (['image/apng', 'image/vnd.mozilla.apng'].includes(type)) {
+ return await convertToApng(path);
+ } else if (type.startsWith('video/')) {
+ return await GenerateVideoThumbnail(path);
+ }
+ }
+
+ return {
+ data: fs.readFileSync(path),
+ ext,
+ type,
+ };
+ };
+
+ const image = await convertFile();
+ ctx.body = image.data;
+ ctx.set('Content-Type', file.type);
+ ctx.set('Cache-Control', 'max-age=31536000, immutable');
+ } catch (e) {
+ serverLogger.error(e);
+
+ if (typeof e == 'number' && e >= 400 && e < 500) {
+ ctx.status = e;
+ ctx.set('Cache-Control', 'max-age=86400');
+ } else {
+ ctx.status = 500;
+ ctx.set('Cache-Control', 'max-age=300');
+ }
+ } finally {
+ cleanup();
+ }
+ return;
+ }
+
ctx.status = 204;
ctx.set('Cache-Control', 'max-age=86400');
return;
}
- const isThumbnail = file.thumbnailAccessKey === key;
- const isWebpublic = file.webpublicAccessKey === key;
-
if (isThumbnail) {
ctx.body = InternalStorage.read(key);
ctx.set('Content-Type', 'image/jpeg');
diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts
index b69fef2afe..877075b8b0 100644
--- a/src/services/drive/add-file.ts
+++ b/src/services/drive/add-file.ts
@@ -424,6 +424,10 @@ export default async function(
file.url = url;
file.thumbnailUrl = url;
file.webpublicUrl = url;
+ // ローカルプロキシ用
+ file.accessKey = uuid();
+ file.thumbnailAccessKey = 'thumbnail-' + uuid();
+ file.webpublicAccessKey = 'webpublic-' + uuid();
}
}
diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts
index 6b17bc313c..18603617d2 100644
--- a/src/services/drive/delete-file.ts
+++ b/src/services/drive/delete-file.ts
@@ -5,6 +5,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '../chart';
import { createDeleteObjectStorageFileJob } from '../../queue';
import { fetchMeta } from '../../misc/fetch-meta';
import { getS3 } from './s3';
+import { v4 as uuid } from 'uuid';
export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) {
@@ -71,6 +72,10 @@ function postProcess(file: DriveFile, isExpired = false) {
thumbnailUrl: file.uri,
webpublicUrl: file.uri,
size: 0,
+ // ローカルプロキシ用
+ accessKey: uuid(),
+ thumbnailAccessKey: 'thumbnail-' + uuid(),
+ webpublicAccessKey: 'webpublic-' + uuid(),
});
} else {
DriveFiles.delete(file.id);