From 385846d43d9f8b8b4efdef1bb13055f8639d155f Mon Sep 17 00:00:00 2001 From: piuvas Date: Thu, 28 Nov 2024 18:47:20 -0300 Subject: [PATCH 01/12] make block confirm dialog localizable. --- locales/index.d.ts | 4 ++++ packages/frontend/src/scripts/get-user-menu.ts | 2 +- sharkey-locales/en-US.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/locales/index.d.ts b/locales/index.d.ts index d1cb1f97ea..f0f8df56bb 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -11000,6 +11000,10 @@ export interface Locale extends ILocale { * Show warning when opening external URLs */ "warnExternalUrl": string; + /** + * Confirm + */ + "confirm": string; "_mfm": { /** * This is not a widespread feature, it may not display properly on most other fedi software, including other Misskey forks diff --git a/packages/frontend/src/scripts/get-user-menu.ts b/packages/frontend/src/scripts/get-user-menu.ts index d15279d633..55189aa481 100644 --- a/packages/frontend/src/scripts/get-user-menu.ts +++ b/packages/frontend/src/scripts/get-user-menu.ts @@ -104,7 +104,7 @@ export function getUserMenu(user: Misskey.entities.UserDetailed, router: IRouter async function getConfirmed(text: string): Promise { const confirm = await os.confirm({ type: 'warning', - title: 'confirm', + title: i18n.ts.confirm, text, }); diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index 163fd0b0ae..e30ce0714f 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -156,6 +156,7 @@ allowClickingNotifications: "Allow clicking on pop-up notifications" pinnedOnly: "Pinned" blockingYou: "Blocking you" warnExternalUrl: "Show warning when opening external URLs" +confirm: "Confirm" _delivery: stop: "Suspend delivery" resume: "Resume delivery" From 51bc393d589eceb60e5e70a9ab4ca1e2d95dcf6b Mon Sep 17 00:00:00 2001 From: piuvas Date: Thu, 28 Nov 2024 19:12:07 -0300 Subject: [PATCH 02/12] remove title and change dialog icon --- locales/index.d.ts | 4 ---- packages/frontend/src/scripts/get-user-menu.ts | 3 +-- sharkey-locales/en-US.yml | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/locales/index.d.ts b/locales/index.d.ts index f0f8df56bb..d1cb1f97ea 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -11000,10 +11000,6 @@ export interface Locale extends ILocale { * Show warning when opening external URLs */ "warnExternalUrl": string; - /** - * Confirm - */ - "confirm": string; "_mfm": { /** * This is not a widespread feature, it may not display properly on most other fedi software, including other Misskey forks diff --git a/packages/frontend/src/scripts/get-user-menu.ts b/packages/frontend/src/scripts/get-user-menu.ts index 55189aa481..090cffe203 100644 --- a/packages/frontend/src/scripts/get-user-menu.ts +++ b/packages/frontend/src/scripts/get-user-menu.ts @@ -103,8 +103,7 @@ export function getUserMenu(user: Misskey.entities.UserDetailed, router: IRouter async function getConfirmed(text: string): Promise { const confirm = await os.confirm({ - type: 'warning', - title: i18n.ts.confirm, + type: 'question', text, }); diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index e30ce0714f..163fd0b0ae 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -156,7 +156,6 @@ allowClickingNotifications: "Allow clicking on pop-up notifications" pinnedOnly: "Pinned" blockingYou: "Blocking you" warnExternalUrl: "Show warning when opening external URLs" -confirm: "Confirm" _delivery: stop: "Suspend delivery" resume: "Resume delivery" From 0efd5eff2b3b503a1cab48205a5240b08a07bacc Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 28 Nov 2024 19:17:34 -0500 Subject: [PATCH 03/12] remove unused import from InternalStorageService --- packages/backend/src/core/InternalStorageService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/InternalStorageService.ts b/packages/backend/src/core/InternalStorageService.ts index f7371f8e79..5b179baf41 100644 --- a/packages/backend/src/core/InternalStorageService.ts +++ b/packages/backend/src/core/InternalStorageService.ts @@ -4,7 +4,7 @@ */ import * as fs from 'node:fs'; -import { copyFile, mkdir, unlink, writeFile } from 'node:fs/promises'; +import { copyFile, unlink, writeFile } from 'node:fs/promises'; import * as Path from 'node:path'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; From 387dc4bb4b57b5071de3f2180c591c42d3ddd5b3 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 23 Nov 2024 17:51:09 +0000 Subject: [PATCH 04/12] UNTESTED maybe laxer match on authority - fixes #815 --- packages/backend/package.json | 4 ++- packages/backend/src/core/UtilityService.ts | 9 ++++++ .../src/core/activitypub/ApRequestService.ts | 2 +- .../src/core/activitypub/ApResolverService.ts | 2 +- .../core/activitypub/models/ApNoteService.ts | 6 ++-- .../activitypub/models/ApPersonService.ts | 16 +++++------ pnpm-lock.yaml | 28 ++++++++++++++----- 7 files changed, 46 insertions(+), 21 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 19547c5033..00baef56d8 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -93,6 +93,7 @@ "@swc/core": "1.6.6", "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.1.1", + "@types/psl": "^1.1.3", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", @@ -135,9 +136,9 @@ "json5": "2.2.3", "jsonld": "8.3.2", "jsrsasign": "11.1.0", + "juice": "11.0.0", "megalodon": "workspace:*", "meilisearch": "0.42.0", - "juice": "11.0.0", "microformats-parser": "2.0.2", "mime-types": "2.1.35", "misskey-js": "workspace:*", @@ -158,6 +159,7 @@ "probe-image-size": "7.2.3", "promise-limit": "2.7.0", "proxy-addr": "^2.0.7", + "psl": "^1.13.0", "pug": "3.0.3", "punycode": "2.3.1", "qrcode": "1.5.4", diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 4c6d539e16..c84e7f212b 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -7,6 +7,7 @@ import { URL } from 'node:url'; import { toASCII } from 'punycode'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; +import psl from 'psl'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; @@ -122,6 +123,14 @@ export class UtilityService { return host; } + @bindThis + public punyHostPSLDomain(url: string): string { + const urlObj = new URL(url); + const domain = psl.get(urlObj.hostname) ?? urlObj.hostname; + const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; + return host; + } + public isFederationAllowedHost(host: string): boolean { if (this.meta.federation === 'none') return false; if (this.meta.federation === 'specified' && !this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`))) return false; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index eeff73385b..0dea615cb0 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -243,7 +243,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href) { - if (this.utilityService.punyHost(url) === this.utilityService.punyHost(href)) { + if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { return await this.signedGet(href, user, false); } } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index d4964d544d..e56b9ebc99 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -131,7 +131,7 @@ export class Resolver { throw new UnrecoverableError(`invalid AP object ${value}: missing id`); } - if (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value)) { + if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) { throw new UnrecoverableError(`invalid AP object ${value}: id ${object.id} has different host`); } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 3d4a33ded2..b8aa67e9ea 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -192,8 +192,8 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${entryUri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { - throw new Error(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { + throw new UnrecoverableError(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); } } @@ -444,7 +444,7 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${noteUri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { throw new UnrecoverableError(`note url <> id host mismatch: ${url} <> ${note.id} in ${noteUri}`); } } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index cd6078b2ed..598486cd84 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -138,7 +138,7 @@ export class ApPersonService implements OnModuleInit { */ @bindThis private validateActor(x: IObject, uri: string): IActor { - const expectHost = this.utilityService.punyHost(uri); + const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`); @@ -152,7 +152,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`); } - const inboxHost = this.utilityService.punyHost(x.inbox); + const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); if (inboxHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); } @@ -160,7 +160,7 @@ export class ApPersonService implements OnModuleInit { const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); if (sharedInboxObject != null) { const sharedInbox = getApId(sharedInboxObject); - if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHost(sharedInbox) === expectHost)) { + if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); } } @@ -170,7 +170,7 @@ export class ApPersonService implements OnModuleInit { if (xCollection != null) { const collectionUri = getApId(xCollection); if (typeof collectionUri === 'string' && collectionUri.length > 0) { - if (this.utilityService.punyHost(collectionUri) !== expectHost) { + if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); } } else if (collectionUri != null) { @@ -202,7 +202,7 @@ export class ApPersonService implements OnModuleInit { x.summary = truncate(x.summary, summaryLength); } - const idHost = this.utilityService.punyHost(x.id); + const idHost = this.utilityService.punyHostPSLDomain(x.id); if (idHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`); } @@ -212,7 +212,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`); } - const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id); + const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id); if (publicKeyIdHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`); } @@ -351,7 +351,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } @@ -563,7 +563,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2d7035550..76e399d5dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: '@twemoji/parser': specifier: 15.1.1 version: 15.1.1 + '@types/psl': + specifier: ^1.1.3 + version: 1.1.3 accepts: specifier: 1.3.8 version: 1.3.8 @@ -365,6 +368,9 @@ importers: proxy-addr: specifier: ^2.0.7 version: 2.0.7 + psl: + specifier: ^1.13.0 + version: 1.13.0 pug: specifier: 3.0.3 version: 3.0.3 @@ -4835,6 +4841,9 @@ packages: '@types/proxy-addr@2.0.3': resolution: {integrity: sha512-TgAHHO4tNG3HgLTUhB+hM4iwW6JUNeQHCLnF1DjaDA9c69PN+IasoFu2MYDhubFc+ZIw5c5t9DMtjvrD6R3Egg==} + '@types/psl@1.1.3': + resolution: {integrity: sha512-Iu174JHfLd7i/XkXY6VDrqSlPvTDQOtQI7wNAXKKOAADJ9TduRLkNdMgjGiMxSttUIZnomv81JAbAbC0DhggxA==} + '@types/pug@2.0.10': resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==} @@ -5307,6 +5316,7 @@ packages: acorn-import-assertions@1.9.0: resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==} + deprecated: package has been renamed to acorn-import-attributes peerDependencies: acorn: ^8 @@ -9736,8 +9746,8 @@ packages: pseudomap@1.0.2: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + psl@1.13.0: + resolution: {integrity: sha512-BFwmFXiJoFqlUpZ5Qssolv15DMyc84gTBds1BjsV1BfXEo1UyyD7GsmN67n7J77uRhoSNW1AXtXKPLcBFQn9Aw==} pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} @@ -13103,7 +13113,7 @@ snapshots: '@eslint/config-array@0.17.1': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13125,7 +13135,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 espree: 10.1.0 globals: 14.0.0 ignore: 5.3.1 @@ -15648,6 +15658,8 @@ snapshots: dependencies: '@types/node': 20.14.12 + '@types/psl@1.1.3': {} + '@types/pug@2.0.10': {} '@types/punycode@2.1.4': {} @@ -18435,7 +18447,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 escape-string-regexp: 4.0.0 eslint-scope: 8.0.2 eslint-visitor-keys: 4.0.0 @@ -22014,7 +22026,9 @@ snapshots: pseudomap@1.0.2: {} - psl@1.9.0: {} + psl@1.13.0: + dependencies: + punycode: 2.3.1 pstree.remy@1.1.8: {} @@ -23256,7 +23270,7 @@ snapshots: tough-cookie@4.1.4: dependencies: - psl: 1.9.0 + psl: 1.13.0 punycode: 2.3.1 universalify: 0.2.0 url-parse: 1.5.10 From fd2af6dfe62dda3549cffe44a735314086279fc0 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 23 Nov 2024 17:51:33 +0000 Subject: [PATCH 05/12] silence linter? it started complaining about that `true &&` all of a sudden --- packages/backend/test/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 26de19eaf1..cf97473d14 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -612,8 +612,8 @@ export async function initTestDb(justBorrow = false, initEntities?: any[]) { username: config.db.user, password: config.db.pass, database: config.db.db, - synchronize: true && !justBorrow, - dropSchema: true && !justBorrow, + synchronize: !justBorrow, + dropSchema: !justBorrow, entities: initEntities ?? entities, }); From 82376f312d2286b8acf8baf5c652ac08e6b10390 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 24 Nov 2024 13:51:44 +0000 Subject: [PATCH 06/12] use "userland" `punycode`, plus tests thanks to CenTdemeern1 for the `import` incantation --- packages/backend/src/core/UtilityService.ts | 6 +-- packages/backend/test/unit/UtilityService.ts | 39 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 packages/backend/test/unit/UtilityService.ts diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index c84e7f212b..fca715164e 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -4,7 +4,7 @@ */ import { URL } from 'node:url'; -import { toASCII } from 'punycode'; +import punycode from 'punycode/'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; import psl from 'psl'; @@ -107,13 +107,13 @@ export class UtilityService { @bindThis public toPuny(host: string): string { - return toASCII(host.toLowerCase()); + return punycode.toASCII(host.toLowerCase()); } @bindThis public toPunyNullable(host: string | null | undefined): string | null { if (host == null) return null; - return toASCII(host.toLowerCase()); + return punycode.toASCII(host.toLowerCase()); } @bindThis diff --git a/packages/backend/test/unit/UtilityService.ts b/packages/backend/test/unit/UtilityService.ts new file mode 100644 index 0000000000..b5e978569e --- /dev/null +++ b/packages/backend/test/unit/UtilityService.ts @@ -0,0 +1,39 @@ +import * as assert from 'assert'; +import { Test } from '@nestjs/testing'; + +import { CoreModule } from '@/core/CoreModule.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { GlobalModule } from '@/GlobalModule.js'; + +describe('UtilityService', () => { + let utilityService: UtilityService; + + beforeAll(async () => { + const app = await Test.createTestingModule({ + imports: [GlobalModule, CoreModule], + }).compile(); + utilityService = app.get(UtilityService); + }); + + describe('punyHost', () => { + test('simple', () => { + assert.equal(utilityService.punyHost('http://www.foo.com'),'www.foo.com'); + }); + test('japanese', () => { + assert.equal(utilityService.punyHost('http://www.新聞.com'),'www.xn--efvv70d.com'); + }); + }); + + describe('punyHostPSLDomain', () => { + test('simple', () => { + assert.equal(utilityService.punyHostPSLDomain('http://www.foo.com'),'foo.com'); + }); + test('japanese', () => { + assert.equal(utilityService.punyHostPSLDomain('http://www.新聞.com'),'xn--efvv70d.com'); + }); + test('lower', () => { + assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'),'foo.github.io'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'),'bar.github.io'); + }); + }); +}); From 97d17c537b3a8522ce23190b6d0273a8c7e6f486 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 24 Nov 2024 14:04:03 +0000 Subject: [PATCH 07/12] spaces / lint --- packages/backend/test/unit/UtilityService.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/test/unit/UtilityService.ts b/packages/backend/test/unit/UtilityService.ts index b5e978569e..837b55206e 100644 --- a/packages/backend/test/unit/UtilityService.ts +++ b/packages/backend/test/unit/UtilityService.ts @@ -17,23 +17,23 @@ describe('UtilityService', () => { describe('punyHost', () => { test('simple', () => { - assert.equal(utilityService.punyHost('http://www.foo.com'),'www.foo.com'); + assert.equal(utilityService.punyHost('http://www.foo.com'), 'www.foo.com'); }); test('japanese', () => { - assert.equal(utilityService.punyHost('http://www.新聞.com'),'www.xn--efvv70d.com'); + assert.equal(utilityService.punyHost('http://www.新聞.com'), 'www.xn--efvv70d.com'); }); }); describe('punyHostPSLDomain', () => { test('simple', () => { - assert.equal(utilityService.punyHostPSLDomain('http://www.foo.com'),'foo.com'); + assert.equal(utilityService.punyHostPSLDomain('http://www.foo.com'), 'foo.com'); }); test('japanese', () => { - assert.equal(utilityService.punyHostPSLDomain('http://www.新聞.com'),'xn--efvv70d.com'); + assert.equal(utilityService.punyHostPSLDomain('http://www.新聞.com'), 'xn--efvv70d.com'); }); test('lower', () => { - assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'),'foo.github.io'); - assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'),'bar.github.io'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'), 'foo.github.io'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'), 'bar.github.io'); }); }); }); From f0139ae1e08b65354ab34fa4d422c41844bf2fdb Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 26 Nov 2024 11:14:51 +0000 Subject: [PATCH 08/12] actually use the correct `import` syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CenTdemeern1 had told me, but I got it wrong ☹ --- packages/backend/src/core/UtilityService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index fca715164e..fc38c9dac6 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -4,7 +4,7 @@ */ import { URL } from 'node:url'; -import punycode from 'punycode/'; +import punycode from 'punycode/punycode.js'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; import psl from 'psl'; From 51afbbaf72eb88e1a1014cdf34c6b064d8490cd2 Mon Sep 17 00:00:00 2001 From: dakkar Date: Wed, 27 Nov 2024 14:44:33 +0000 Subject: [PATCH 09/12] handle `.masto.host` specially --- packages/backend/src/core/UtilityService.ts | 14 +++++++++++++- packages/backend/test/unit/UtilityService.ts | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index fc38c9dac6..dda4bd5fbd 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -123,10 +123,22 @@ export class UtilityService { return host; } + private specialSuffix(hostname: string): string | null { + // masto.host provides domain names for its clients, we have to + // treat it as if it were a public suffix + const mastoHost = hostname.match(/\.?([a-zA-Z0-9-]+\.masto\.host)$/i); + if (mastoHost) { + return mastoHost[1]; + } + + return null; + } + @bindThis public punyHostPSLDomain(url: string): string { const urlObj = new URL(url); - const domain = psl.get(urlObj.hostname) ?? urlObj.hostname; + const hostname = urlObj.hostname; + const domain = this.specialSuffix(hostname) ?? psl.get(hostname) ?? hostname; const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; return host; } diff --git a/packages/backend/test/unit/UtilityService.ts b/packages/backend/test/unit/UtilityService.ts index 837b55206e..d86e794f2f 100644 --- a/packages/backend/test/unit/UtilityService.ts +++ b/packages/backend/test/unit/UtilityService.ts @@ -35,5 +35,9 @@ describe('UtilityService', () => { assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'), 'foo.github.io'); assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'), 'bar.github.io'); }); + test('special', () => { + assert.equal(utilityService.punyHostPSLDomain('http://foo.masto.host'), 'foo.masto.host'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.masto.host'), 'bar.masto.host'); + }); }); }); From e6e48fb6bcdc06cfd91357717c9bef1d930c2a1a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 3 Nov 2024 17:03:24 -0500 Subject: [PATCH 10/12] interpret Dislike activities as Undo(Like) --- .../backend/src/core/activitypub/ApInboxService.ts | 13 ++++++++++--- packages/backend/src/core/activitypub/type.ts | 5 +++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 5d0404d24e..75ba5e97b9 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -32,7 +32,7 @@ import { AbuseReportService } from '@/core/AbuseReportService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -41,7 +41,7 @@ import { ApAudienceService } from './ApAudienceService.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; import type { Resolver } from './ApResolverService.js'; -import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js'; +import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IDislike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js'; @Injectable() export class ApInboxService { @@ -166,6 +166,8 @@ export class ApInboxService { return await this.announce(actor, activity, resolver); } else if (isLike(activity)) { return await this.like(actor, activity, resolver); + } else if (isDislike(activity)) { + return await this.dislike(actor, activity); } else if (isUndo(activity)) { return await this.undo(actor, activity, resolver); } else if (isBlock(activity)) { @@ -220,6 +222,11 @@ export class ApInboxService { } } + @bindThis + private async dislike(actor: MiRemoteUser, dislike: IDislike): Promise { + return await this.undoLike(actor, dislike); + } + @bindThis private async accept(actor: MiRemoteUser, activity: IAccept, resolver?: Resolver): Promise { const uri = activity.id ?? activity; @@ -782,7 +789,7 @@ export class ApInboxService { } @bindThis - private async undoLike(actor: MiRemoteUser, activity: ILike): Promise { + private async undoLike(actor: MiRemoteUser, activity: ILike | IDislike): Promise { const targetUri = getApId(activity.object); const note = await this.apNoteService.fetchNote(targetUri); diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 85ddc20064..c9e20e0168 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -336,6 +336,10 @@ export interface ILike extends IActivity { _misskey_reaction?: string; } +export interface IDislike extends IActivity { + type: 'Dislike'; +} + export interface IAnnounce extends IActivity { type: 'Announce'; } @@ -368,6 +372,7 @@ export const isLike = (object: IObject): object is ILike => { const type = getApType(object); return type != null && ['Like', 'EmojiReaction', 'EmojiReact'].includes(type); }; +export const isDislike = (object: IObject): object is IDislike => getApType(object) === 'Dislike'; export const isAnnounce = (object: IObject): object is IAnnounce => getApType(object) === 'Announce'; export const isBlock = (object: IObject): object is IBlock => getApType(object) === 'Block'; export const isFlag = (object: IObject): object is IFlag => getApType(object) === 'Flag'; From 3d3cf5bd7aa19aaf245aa1fe426fe39f2e7c3436 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 28 Nov 2024 19:56:26 -0500 Subject: [PATCH 11/12] add option `filePermissionBits` to override permissions on locally-stored files This is useful for custom deployments, such as using a reverse proxy to serve static files directly --- .config/ci.yml | 5 +++++ .config/cypress-devcontainer.yml | 5 +++++ .config/docker_example.yml | 5 +++++ .config/example.yml | 5 +++++ packages/backend/src/config.ts | 10 ++++++++-- packages/backend/src/core/InternalStorageService.ts | 12 ++++++++++-- 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index d20ede8d35..8730ccab3a 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -229,3 +229,8 @@ checkActivityPubGetSignature: false # Upload or download file size limits (bytes) #maxFileSize: 262144000 + +# CHMod-style permission bits to apply to uploaded files. +# Permission bits are specified as a base-8 string representing User/Group/Other permissions. +# This setting is only useful for custom deployments, such as using a reverse proxy to serve media. +#filePermissionBits: '644' diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index d8013a1c95..342b0f43da 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -222,3 +222,8 @@ allowedPrivateNetworks: [ # Upload or download file size limits (bytes) #maxFileSize: 262144000 + +# CHMod-style permission bits to apply to uploaded files. +# Permission bits are specified as a base-8 string representing User/Group/Other permissions. +# This setting is only useful for custom deployments, such as using a reverse proxy to serve media. +#filePermissionBits: '644' diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 5fac3dc41e..ce2daf3aec 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -312,3 +312,8 @@ checkActivityPubGetSignature: false # Upload or download file size limits (bytes) #maxFileSize: 262144000 + +# CHMod-style permission bits to apply to uploaded files. +# Permission bits are specified as a base-8 string representing User/Group/Other permissions. +# This setting is only useful for custom deployments, such as using a reverse proxy to serve media. +#filePermissionBits: '644' diff --git a/.config/example.yml b/.config/example.yml index cf7b972de5..9debb3bf70 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -334,3 +334,8 @@ checkActivityPubGetSignature: false # PID File of master process #pidFile: /tmp/misskey.pid + +# CHMod-style permission bits to apply to uploaded files. +# Permission bits are specified as a base-8 string representing User/Group/Other permissions. +# This setting is only useful for custom deployments, such as using a reverse proxy to serve media. +#filePermissionBits: '644' diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 3dc49c7eb6..9cac058d96 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -115,6 +115,7 @@ type Source = { }; pidFile: string; + filePermissionBits?: string; }; export type Config = { @@ -212,6 +213,7 @@ export type Config = { } | undefined; pidFile: string; + filePermissionBits?: string; }; const _filename = fileURLToPath(import.meta.url); @@ -347,6 +349,7 @@ export function loadConfig(): Config { deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7), import: config.import, pidFile: config.pidFile, + filePermissionBits: config.filePermissionBits, }; } @@ -452,7 +455,10 @@ function applyEnvOverrides(config: Source) { } } - const alwaysStrings = { 'chmodSocket': true } as { [key: string]: boolean }; + const alwaysStrings: { [key in string]?: boolean } = { + 'chmodSocket': true, + 'filePermissionBits': true, + }; function _assign(path: (string | number)[], lastStep: string | number, value: string) { let thisConfig = config as any; @@ -490,7 +496,7 @@ function applyEnvOverrides(config: Source) { _apply_top(['sentryForBackend', 'enableNodeProfiling']); _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]); - _apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile']]); + _apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]); _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature']]); } diff --git a/packages/backend/src/core/InternalStorageService.ts b/packages/backend/src/core/InternalStorageService.ts index 5b179baf41..b00c5796d2 100644 --- a/packages/backend/src/core/InternalStorageService.ts +++ b/packages/backend/src/core/InternalStorageService.ts @@ -4,7 +4,7 @@ */ import * as fs from 'node:fs'; -import { copyFile, unlink, writeFile } from 'node:fs/promises'; +import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises'; import * as Path from 'node:path'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; @@ -41,12 +41,20 @@ export class InternalStorageService { @bindThis public async saveFromPath(key: string, srcPath: string): Promise { await copyFile(srcPath, this.resolvePath(key)); - return `${this.config.url}/files/${key}`; + return await this.finalizeSavedFile(key); } @bindThis public async saveFromBuffer(key: string, data: Buffer): Promise { await writeFile(this.resolvePath(key), data); + return await this.finalizeSavedFile(key); + } + + private async finalizeSavedFile(key: string): Promise { + if (this.config.filePermissionBits) { + const path = this.resolvePath(key); + await chmod(path, this.config.filePermissionBits); + } return `${this.config.url}/files/${key}`; } From b4b72ab2dfc5f77181ece47c01e3fa59e2b58460 Mon Sep 17 00:00:00 2001 From: CenTdemeern1 Date: Sat, 30 Nov 2024 11:42:23 +0100 Subject: [PATCH 12/12] Opinionated changes to merge/issue templates In hopes of making them look a bit nicer by default. --- .gitlab/issue_templates/bug.md | 28 +++++++++++++--------- .gitlab/issue_templates/feature.md | 14 +++++++---- .gitlab/merge_request_templates/default.md | 7 +++--- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.gitlab/issue_templates/bug.md b/.gitlab/issue_templates/bug.md index 6914647570..a909067269 100644 --- a/.gitlab/issue_templates/bug.md +++ b/.gitlab/issue_templates/bug.md @@ -3,27 +3,33 @@ 🔒 Found a security vulnerability? [Please disclose it responsibly.](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/SECURITY.md) 🤝 By submitting this feature request, you agree to follow our [Contribution Guidelines.](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) --> -**What happened?** _(Please give us a brief description of what happened.)_ +# **What happened?** + -**What did you expect to happen?** _(Please give us a brief description of what you expected to happen.)_ +# **What did you expect to happen?** + -**Version** _(What version of Sharkey is your instance running? You can find this by clicking your instance's logo at the top left and then clicking instance information.)_ +# **Version** + -**Instance** _(What instance of Sharkey are you using?)_ +# **Instance** + -**What type of issue is this?** _(If this happens on your device and has to do with the user interface, it's client-side. If this happens on either with the API or the backend, or you got a server-side error in the client, it's server-side.)_ +# **What type of issue is this?** + -**What browser are you using? (Client-side issues only)** +# **What browser are you using? (Client-side issues only)** -**What operating system are you using? (Client-side issues only)** +# **What operating system are you using? (Client-side issues only)** -**How do you deploy Sharkey on your server? (Server-side issues only)** +# **How do you deploy Sharkey on your server? (Server-side issues only)** -**What operating system are you using? (Server-side issues only)** +# **What operating system are you using? (Server-side issues only)** -**Relevant log output** _(Please copy and paste any relevant log output. You can find your log by inspecting the page, and going to the "console" tab. This will be automatically formatted into code, so no need for backticks.)_ +# **Relevant log output** + -**Contribution Guidelines** +# **Contribution Guidelines** By submitting this issue, you agree to follow our [Contribution Guidelines](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) - [ ] I agree to follow this project's Contribution Guidelines - [ ] I have searched the issue tracker for similar issues, and this is not a duplicate. diff --git a/.gitlab/issue_templates/feature.md b/.gitlab/issue_templates/feature.md index d4235eb5a3..a77f9335fe 100644 --- a/.gitlab/issue_templates/feature.md +++ b/.gitlab/issue_templates/feature.md @@ -3,15 +3,19 @@ 🔒 Found a security vulnerability? [Please disclose it responsibly.](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/SECURITY.md) 🤝 By submitting this feature request, you agree to follow our [Contribution Guidelines.](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) --> -**What feature would you like implemented?** _(Please give us a brief description of what you'd like.)_ +# **What feature would you like implemented?** + -**Why should we add this feature?** _(Please give us a brief description of why your feature is important.)_ +# **Why should we add this feature?** + -**Version** _(What version of Sharkey is your instance running? You can find this by clicking your instance's logo at the top left and then clicking instance information.)_ +# **Version** + -**Instance** _(What instance of Sharkey are you using?)_ +# **Instance** + -**Contribution Guidelines** +# **Contribution Guidelines** By submitting this issue, you agree to follow our [Contribution Guidelines](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) - [ ] I agree to follow this project's Contribution Guidelines - [ ] I have searched the issue tracker for similar requests, and this is not a duplicate. diff --git a/.gitlab/merge_request_templates/default.md b/.gitlab/merge_request_templates/default.md index 18bffa5419..e6977def70 100644 --- a/.gitlab/merge_request_templates/default.md +++ b/.gitlab/merge_request_templates/default.md @@ -1,11 +1,12 @@ -**What does this PR do?** _(Please give us a brief description of what this PR does.)_ +# **What does this MR do?** + -**Contribution Guidelines** +# **Contribution Guidelines** By submitting this merge request, you agree to follow our [Contribution Guidelines](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) - [ ] I agree to follow this project's Contribution Guidelines -- [ ] I have made sure to test this pull request +- [ ] I have made sure to test this merge request