From 36b9a0d42fa4bd7036d5d4a11203006d0ceb59f0 Mon Sep 17 00:00:00 2001
From: MeiMei <30769358+mei23@users.noreply.github.com>
Date: Sun, 12 Apr 2020 20:32:34 +0900
Subject: [PATCH] =?UTF-8?q?=E3=83=97=E3=83=AD=E3=82=AD=E3=82=B7=E3=81=AE?=
=?UTF-8?q?=E9=99=A4=E5=A4=96=E3=83=9B=E3=82=B9=E3=83=88=20(#6244)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* プロキシの除外ホスト
* オブジェクトストレージとの通信にProxyを使うかを選択できるように
* fix lint
* コメント
Co-authored-by: rinsuki <428rinsuki+git@gmail.com>
---
.config/example.yml | 5 ++
locales/ja-JP.yml | 2 +
.../1586624197029-AddObjectStorageUseProxy.ts | 14 ++++++
src/client/pages/instance/settings.vue | 4 ++
src/config/types.ts | 1 +
src/misc/download-url.ts | 4 +-
src/misc/fetch.ts | 49 +++++++++++++++----
src/models/entities/meta.ts | 5 ++
src/remote/activitypub/request.ts | 4 +-
src/server/api/endpoints/admin/update-meta.ts | 8 +++
src/server/api/endpoints/meta.ts | 1 +
src/services/drive/s3.ts | 8 ++-
12 files changed, 89 insertions(+), 16 deletions(-)
create mode 100644 migration/1586624197029-AddObjectStorageUseProxy.ts
diff --git a/.config/example.yml b/.config/example.yml
index 201082cce9..1794dc9a8b 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -142,6 +142,11 @@ id: 'aid'
# Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128
+#proxyBypassHosts: [
+# 'example.com',
+# '192.0.2.8'
+#]
+
# Proxy for SMTP/SMTPS
#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index 29a4c2d3d6..0827091532 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -454,6 +454,8 @@ objectStorageRegion: "Region"
objectStorageRegionDesc: "'xx-east-1'のようなregionを指定してください。使用サービスにregionの概念がない場合は、空または'us-east-1'にしてください。"
objectStorageUseSSL: "SSLを使用する"
objectStorageUseSSLDesc: "API接続にhttpsを使用しない場合はオフにしてください"
+objectStorageUseProxy: "Proxyを利用する"
+objectStorageUseProxyDesc: "API接続にproxyを利用しない場合はオフにしてください"
serverLogs: "サーバーログ"
deleteAll: "全て削除"
showFixedPostForm: "タイムライン上部に投稿フォームを表示する"
diff --git a/migration/1586624197029-AddObjectStorageUseProxy.ts b/migration/1586624197029-AddObjectStorageUseProxy.ts
new file mode 100644
index 0000000000..deadf94834
--- /dev/null
+++ b/migration/1586624197029-AddObjectStorageUseProxy.ts
@@ -0,0 +1,14 @@
+import {MigrationInterface, QueryRunner} from 'typeorm';
+
+export class AddObjectStorageUseProxy1586624197029 implements MigrationInterface {
+ name = 'AddObjectStorageUseProxy1586624197029'
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseProxy" boolean NOT NULL DEFAULT true`, undefined);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseProxy"`, undefined);
+ }
+
+}
diff --git a/src/client/pages/instance/settings.vue b/src/client/pages/instance/settings.vue
index f0a123f279..f7db4aa10b 100644
--- a/src/client/pages/instance/settings.vue
+++ b/src/client/pages/instance/settings.vue
@@ -116,6 +116,7 @@
<mk-input v-model="objectStorageSecretKey" :disabled="!useObjectStorage"><template #icon><fa :icon="faKey"/></template>Secret key</mk-input>
</div>
<mk-switch v-model="objectStorageUseSSL" :disabled="!useObjectStorage">{{ $t('objectStorageUseSSL') }}<template #desc>{{ $t('objectStorageUseSSLDesc') }}</template></mk-switch>
+ <mk-switch v-model="objectStorageUseProxy" :disabled="!useObjectStorage">{{ $t('objectStorageUseProxy') }}<template #desc>{{ $t('objectStorageUseProxyDesc') }}</template></mk-switch>
</template>
</div>
<div class="_footer">
@@ -249,6 +250,7 @@ export default Vue.extend({
objectStorageAccessKey: null,
objectStorageSecretKey: null,
objectStorageUseSSL: false,
+ objectStorageUseProxy: false,
enableTwitterIntegration: false,
twitterConsumerKey: null,
twitterConsumerSecret: null,
@@ -303,6 +305,7 @@ export default Vue.extend({
this.objectStorageAccessKey = this.meta.objectStorageAccessKey;
this.objectStorageSecretKey = this.meta.objectStorageSecretKey;
this.objectStorageUseSSL = this.meta.objectStorageUseSSL;
+ this.objectStorageUseProxy = this.meta.objectStorageUseProxy;
this.enableTwitterIntegration = this.meta.enableTwitterIntegration;
this.twitterConsumerKey = this.meta.twitterConsumerKey;
this.twitterConsumerSecret = this.meta.twitterConsumerSecret;
@@ -411,6 +414,7 @@ export default Vue.extend({
objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null,
objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null,
objectStorageUseSSL: this.objectStorageUseSSL,
+ objectStorageUseProxy: this.objectStorageUseProxy,
enableTwitterIntegration: this.enableTwitterIntegration,
twitterConsumerKey: this.twitterConsumerKey,
twitterConsumerSecret: this.twitterConsumerSecret,
diff --git a/src/config/types.ts b/src/config/types.ts
index a33901bde6..4f025750b0 100644
--- a/src/config/types.ts
+++ b/src/config/types.ts
@@ -35,6 +35,7 @@ export type Source = {
proxy?: string;
proxySmtp?: string;
+ proxyBypassHosts?: string[];
accesslog?: string;
diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts
index 3f42fb3bef..9c8439f2c0 100644
--- a/src/misc/download-url.ts
+++ b/src/misc/download-url.ts
@@ -2,7 +2,7 @@ import * as fs from 'fs';
import * as stream from 'stream';
import * as util from 'util';
import fetch from 'node-fetch';
-import { httpAgent, httpsAgent } from './fetch';
+import { getAgentByUrl } from './fetch';
import { AbortController } from 'abort-controller';
import config from '../config';
import * as chalk from 'chalk';
@@ -25,7 +25,7 @@ export async function downloadUrl(url: string, path: string) {
},
timeout: 10 * 1000,
signal: controller.signal,
- agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+ agent: getAgentByUrl,
});
if (!response.ok) {
diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts
index 887aae1659..358bc25030 100644
--- a/src/misc/fetch.ts
+++ b/src/misc/fetch.ts
@@ -13,7 +13,7 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
Accept: accept
}, headers || {}),
timeout,
- agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+ agent: getAgentByUrl,
});
if (!res.ok) {
@@ -27,17 +27,46 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
return await res.json();
}
+/**
+ * Get http non-proxy agent
+ */
+const _http = new http.Agent({
+ keepAlive: true,
+ keepAliveMsecs: 30 * 1000,
+});
+
+/**
+ * Get https non-proxy agent
+ */
+const _https = new https.Agent({
+ keepAlive: true,
+ keepAliveMsecs: 30 * 1000,
+ lookup: cache.lookup,
+});
+
+/**
+ * Get http proxy or non-proxy agent
+ */
export const httpAgent = config.proxy
? new HttpProxyAgent(config.proxy)
- : new http.Agent({
- keepAlive: true,
- keepAliveMsecs: 30 * 1000,
- });
+ : _http;
+/**
+ * Get https proxy or non-proxy agent
+ */
export const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
- : new https.Agent({
- keepAlive: true,
- keepAliveMsecs: 30 * 1000,
- lookup: cache.lookup,
- });
+ : _https;
+
+/**
+ * Get agent by URL
+ * @param url URL
+ * @param bypassProxy Allways bypass proxy
+ */
+export function getAgentByUrl(url: URL, bypassProxy = false) {
+ if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
+ return url.protocol == 'http:' ? _http : _https;
+ } else {
+ return url.protocol == 'http:' ? httpAgent : httpsAgent;
+ }
+}
diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts
index ee62cb24ba..bb463c52f4 100644
--- a/src/models/entities/meta.ts
+++ b/src/models/entities/meta.ts
@@ -348,4 +348,9 @@ export class Meta {
default: true,
})
public objectStorageUseSSL: boolean;
+
+ @Column('boolean', {
+ default: true,
+ })
+ public objectStorageUseProxy: boolean;
}
diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts
index 24540827b2..ab3ae025a5 100644
--- a/src/remote/activitypub/request.ts
+++ b/src/remote/activitypub/request.ts
@@ -6,7 +6,7 @@ import config from '../../config';
import { ILocalUser } from '../../models/entities/user';
import { UserKeypairs } from '../../models';
import { ensure } from '../../prelude/ensure';
-import { httpsAgent } from '../../misc/fetch';
+import { getAgentByUrl } from '../../misc/fetch';
export default async (user: ILocalUser, url: string, object: any) => {
const timeout = 10 * 1000;
@@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
await new Promise((resolve, reject) => {
const req = https.request({
- agent: httpsAgent,
+ agent: getAgentByUrl(new URL(`https://example.net`)),
protocol,
hostname,
port,
diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts
index e9527d9ac5..1bc20029ef 100644
--- a/src/server/api/endpoints/admin/update-meta.ts
+++ b/src/server/api/endpoints/admin/update-meta.ts
@@ -394,6 +394,10 @@ export const meta = {
objectStorageUseSSL: {
validator: $.optional.bool
},
+
+ objectStorageUseProxy: {
+ validator: $.optional.bool
+ }
}
};
@@ -632,6 +636,10 @@ export default define(meta, async (ps, me) => {
set.objectStorageUseSSL = ps.objectStorageUseSSL;
}
+ if (ps.objectStorageUseProxy !== undefined) {
+ set.objectStorageUseProxy = ps.objectStorageUseProxy;
+ }
+
await getConnection().transaction(async transactionalEntityManager => {
const meta = await transactionalEntityManager.findOne(Meta, {
order: {
diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts
index 280a6fbae6..179355489b 100644
--- a/src/server/api/endpoints/meta.ts
+++ b/src/server/api/endpoints/meta.ts
@@ -190,6 +190,7 @@ export default define(meta, async (ps, me) => {
response.objectStorageAccessKey = instance.objectStorageAccessKey;
response.objectStorageSecretKey = instance.objectStorageSecretKey;
response.objectStorageUseSSL = instance.objectStorageUseSSL;
+ response.objectStorageUseProxy = instance.objectStorageUseProxy;
}
return response;
diff --git a/src/services/drive/s3.ts b/src/services/drive/s3.ts
index 2cbeef106d..abe3c166a5 100644
--- a/src/services/drive/s3.ts
+++ b/src/services/drive/s3.ts
@@ -1,8 +1,12 @@
import * as S3 from 'aws-sdk/clients/s3';
import { Meta } from '../../models/entities/meta';
-import { httpsAgent, httpAgent } from '../../misc/fetch';
+import { getAgentByUrl } from '../../misc/fetch';
export function getS3(meta: Meta) {
+ const u = meta.objectStorageEndpoint != null
+ ? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
+ : `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
+
return new S3({
endpoint: meta.objectStorageEndpoint || undefined,
accessKeyId: meta.objectStorageAccessKey!,
@@ -11,7 +15,7 @@ export function getS3(meta: Meta) {
sslEnabled: meta.objectStorageUseSSL,
s3ForcePathStyle: !!meta.objectStorageEndpoint,
httpOptions: {
- agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
+ agent: getAgentByUrl(new URL(u), !meta.objectStorageUseProxy)
}
});
}