diff --git a/packages/backend/package.json b/packages/backend/package.json index f56a737eea..f8743e4db9 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -96,6 +96,7 @@ "@swc/cli": "0.3.12", "@swc/core": "1.9.2", "@twemoji/parser": "15.1.1", + "@types/psl": "^1.1.3", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", @@ -157,6 +158,7 @@ "pkce-challenge": "4.1.0", "probe-image-size": "7.2.3", "promise-limit": "2.7.0", + "psl": "^1.13.0", "pug": "3.0.3", "punycode": "2.3.1", "qrcode": "1.5.4", diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 9a2ba72ed3..695fe20be4 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -7,6 +7,7 @@ import { URL } from 'node:url'; import { toASCII } from 'punycode'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; +import psl from 'psl'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; @@ -122,6 +123,14 @@ export class UtilityService { return host; } + @bindThis + public punyHostPSLDomain(url: string): string { + const urlObj = new URL(url); + const domain = psl.get(urlObj.hostname) ?? urlObj.hostname; + const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; + return host; + } + @bindThis public isFederationAllowedHost(host: string): boolean { if (this.meta.federation === 'none') return false; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 8c3b7295e4..5ab4975ead 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -242,8 +242,10 @@ export class ApRequestService { const alternate = document.querySelector('head > link[rel="alternate"][type="application/activity+json"]'); if (alternate) { const href = alternate.getAttribute('href'); - if (href && this.utilityService.punyHost(url) === this.utilityService.punyHost(href)) { - return await this.signedGet(href, user, false); + if (href) { + if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { + return await this.signedGet(href, user, false); + } } } } catch (e) { diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index b0b35274ea..fcaf0b6dca 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -126,7 +126,7 @@ export class Resolver { throw new Error('invalid AP object: missing id'); } - if (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value)) { + if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) { throw new Error(`invalid AP object ${value}: id ${object.id} has different host`); } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index eb2e771a38..c6c6b51d12 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -159,8 +159,8 @@ export class ApNoteService { throw new Error('unexpected schema of note url: ' + url); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { - throw new Error(`note url & uri host mismatch: note url: ${url}, note uri: ${note.id}`); + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { + throw new Error(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); } } @@ -340,7 +340,7 @@ export class ApNoteService { this.logger.info('The note is already inserted while creating itself, reading again'); const duplicate = await this.fetchNote(value); if (!duplicate) { - throw new Error('The note creation failed with duplication error even when there is no duplication'); + throw new Error(`The note creation failed with duplication error even when there is no duplication: ${entryUri}`); } return duplicate; } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 8590861ca0..056629a867 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -136,7 +136,7 @@ export class ApPersonService implements OnModuleInit { */ @bindThis private validateActor(x: IObject, uri: string): IActor { - const expectHost = this.utilityService.punyHost(uri); + const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { throw new Error(`invalid Actor type '${x.type}'`); @@ -150,15 +150,16 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: wrong inbox'); } - if (this.utilityService.punyHost(x.inbox) !== expectHost) { - throw new Error('invalid Actor: inbox has different host'); + const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); + if (inboxHost !== expectHost) { + throw new Error(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); } const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); if (sharedInboxObject != null) { const sharedInbox = getApId(sharedInboxObject); - if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHost(sharedInbox) === expectHost)) { - throw new Error('invalid Actor: wrong shared inbox'); + if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { + throw new Error(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); } } @@ -167,8 +168,8 @@ export class ApPersonService implements OnModuleInit { if (xCollection != null) { const collectionUri = getApId(xCollection); if (typeof collectionUri === 'string' && collectionUri.length > 0) { - if (this.utilityService.punyHost(collectionUri) !== expectHost) { - throw new Error(`invalid Actor: ${collection} has different host`); + if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { + throw new Error(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); } } else if (collectionUri != null) { throw new Error(`invalid Actor: wrong ${collection}`); @@ -199,7 +200,7 @@ export class ApPersonService implements OnModuleInit { x.summary = truncate(x.summary, summaryLength); } - const idHost = this.utilityService.punyHost(x.id); + const idHost = this.utilityService.punyHostPSLDomain(x.id); if (idHost !== expectHost) { throw new Error('invalid Actor: id has different host'); } @@ -209,7 +210,7 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: publicKey.id is not a string'); } - const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id); + const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id); if (publicKeyIdHost !== expectHost) { throw new Error('invalid Actor: publicKey.id has different host'); } @@ -257,7 +258,7 @@ export class ApPersonService implements OnModuleInit { if (Array.isArray(img)) { img = img.find(item => item && item.url) ?? null; } - + // if we have an explicitly missing image, return an // explicitly-null set of values if ((img == null) || (typeof img === 'object' && img.url == null)) { @@ -349,8 +350,8 @@ export class ApPersonService implements OnModuleInit { throw new Error('unexpected schema of person url: ' + url); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { - throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id}`); + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { + throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } @@ -552,8 +553,8 @@ export class ApPersonService implements OnModuleInit { throw new Error('unexpected schema of person url: ' + url); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { - throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id}`); + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { + throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0ec950181..b7e0aee135 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -142,7 +142,7 @@ importers: version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1) '@nestjs/testing': specifier: 10.4.7 - version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)) + version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7) '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -170,6 +170,9 @@ importers: '@twemoji/parser': specifier: 15.1.1 version: 15.1.1 + '@types/psl': + specifier: ^1.1.3 + version: 1.1.3 accepts: specifier: 1.3.8 version: 1.3.8 @@ -353,6 +356,9 @@ importers: promise-limit: specifier: 2.7.0 version: 2.7.0 + psl: + specifier: ^1.13.0 + version: 1.13.0 pug: specifier: 3.0.3 version: 3.0.3 @@ -1163,7 +1169,7 @@ importers: version: 7.17.0(eslint@9.14.0)(typescript@5.6.3) '@vitest/coverage-v8': specifier: 1.6.0 - version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0)) + version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.8)(utf-8-validate@6.0.4))(sass@1.79.4)(terser@5.36.0)) '@vue/runtime-core': specifier: 3.5.12 version: 3.5.12 @@ -4486,6 +4492,9 @@ packages: '@types/prop-types@15.7.5': resolution: {integrity: sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==} + '@types/psl@1.1.3': + resolution: {integrity: sha512-Iu174JHfLd7i/XkXY6VDrqSlPvTDQOtQI7wNAXKKOAADJ9TduRLkNdMgjGiMxSttUIZnomv81JAbAbC0DhggxA==} + '@types/pug@2.0.10': resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==} @@ -9335,8 +9344,8 @@ packages: pseudomap@1.0.2: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + psl@1.13.0: + resolution: {integrity: sha512-BFwmFXiJoFqlUpZ5Qssolv15DMyc84gTBds1BjsV1BfXEo1UyyD7GsmN67n7J77uRhoSNW1AXtXKPLcBFQn9Aw==} pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} @@ -11780,7 +11789,7 @@ snapshots: '@babel/traverse': 7.23.5 '@babel/types': 7.24.7 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -11800,7 +11809,7 @@ snapshots: '@babel/traverse': 7.24.7 '@babel/types': 7.24.7 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12059,7 +12068,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.22.6 '@babel/parser': 7.25.6 '@babel/types': 7.24.7 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12074,7 +12083,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.24.7 '@babel/parser': 7.25.6 '@babel/types': 7.25.6 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12465,7 +12474,7 @@ snapshots: '@eslint/config-array@0.18.0': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -12475,7 +12484,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) espree: 10.3.0 globals: 14.0.0 ignore: 5.3.1 @@ -13180,7 +13189,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7))': + '@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7)': dependencies: '@nestjs/common': 10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1) '@nestjs/core': 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1) @@ -15068,6 +15077,8 @@ snapshots: '@types/prop-types@15.7.5': {} + '@types/psl@1.1.3': {} + '@types/pug@2.0.10': {} '@types/punycode@2.1.4': {} @@ -15384,7 +15395,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0))': + '@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.8)(utf-8-validate@6.0.4))(sass@1.79.4)(terser@5.36.0))': dependencies: '@ampproject/remapping': 2.2.1 '@bcoe/v8-coverage': 0.2.3 @@ -15399,7 +15410,7 @@ snapshots: std-env: 3.7.0 strip-literal: 2.1.0 test-exclude: 6.0.0 - vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0) + vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.8)(utf-8-validate@6.0.4))(sass@1.79.4)(terser@5.36.0) transitivePeerDependencies: - supports-color @@ -15637,14 +15648,14 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color optional: true agent-base@7.1.0: dependencies: - debug: 4.3.5 + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -17248,7 +17259,7 @@ snapshots: esbuild-register@3.5.0(esbuild@0.24.0): dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) esbuild: 0.24.0 transitivePeerDependencies: - supports-color @@ -17490,7 +17501,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) escape-string-regexp: 4.0.0 eslint-scope: 8.2.0 eslint-visitor-keys: 4.2.0 @@ -17935,7 +17946,7 @@ snapshots: follow-redirects@1.15.9(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) for-each@0.3.3: dependencies: @@ -18444,7 +18455,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.5 + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color optional: true @@ -18805,7 +18816,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -19215,35 +19226,6 @@ snapshots: jsdoc-type-pratt-parser@4.1.0: {} - jsdom@24.1.1: - dependencies: - cssstyle: 4.0.1 - data-urls: 5.0.0 - decimal.js: 10.4.3 - form-data: 4.0.1 - html-encoding-sniffer: 4.0.0 - http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.5 - is-potential-custom-element-name: 1.0.1 - nwsapi: 2.2.12 - parse5: 7.2.1 - rrweb-cssom: 0.7.1 - saxes: 6.0.0 - symbol-tree: 3.2.4 - tough-cookie: 4.1.4 - w3c-xmlserializer: 5.0.0 - webidl-conversions: 7.0.0 - whatwg-encoding: 3.1.1 - whatwg-mimetype: 4.0.0 - whatwg-url: 14.0.0 - ws: 8.18.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) - xml-name-validator: 5.0.0 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - optional: true - jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3): dependencies: cssstyle: 4.0.1 @@ -19936,7 +19918,7 @@ snapshots: micromark@4.0.0: dependencies: '@types/debug': 4.1.12 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) decode-named-character-reference: 1.0.2 devlop: 1.1.0 micromark-core-commonmark: 2.0.0 @@ -21045,7 +21027,9 @@ snapshots: pseudomap@1.0.2: {} - psl@1.9.0: {} + psl@1.13.0: + dependencies: + punycode: 2.3.1 pstree.remy@1.1.8: {} @@ -21396,7 +21380,7 @@ snapshots: require-in-the-middle@7.3.0: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) module-details-from-path: 1.0.3 resolve: 1.22.8 transitivePeerDependencies: @@ -21821,7 +21805,7 @@ snapshots: socks-proxy-agent@8.0.2: dependencies: agent-base: 7.1.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -21930,7 +21914,7 @@ snapshots: arg: 5.0.2 bluebird: 3.7.2 check-more-types: 2.24.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7(supports-color@8.1.1) execa: 5.1.1 lazy-ass: 1.6.0 ps-tree: 1.2.0 @@ -22280,12 +22264,12 @@ snapshots: tough-cookie@2.5.0: dependencies: - psl: 1.9.0 + psl: 1.13.0 punycode: 2.3.1 tough-cookie@4.1.4: dependencies: - psl: 1.9.0 + psl: 1.13.0 punycode: 2.3.1 universalify: 0.2.0 url-parse: 1.5.10 @@ -22777,7 +22761,7 @@ snapshots: - supports-color - terser - vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0): + vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.8)(utf-8-validate@6.0.4))(sass@1.79.4)(terser@5.36.0): dependencies: '@vitest/expect': 1.6.0 '@vitest/runner': 1.6.0 @@ -22802,7 +22786,7 @@ snapshots: optionalDependencies: '@types/node': 22.9.0 happy-dom: 10.0.3 - jsdom: 24.1.1 + jsdom: 24.1.1(bufferutil@4.0.8)(utf-8-validate@6.0.4) transitivePeerDependencies: - less - lightningcss