mirror of
https://activitypub.software/TransFem-org/Sharkey.git
synced 2024-12-13 17:30:45 +01:00
upd: rip out AiService
This commit is contained in:
parent
08de1f7baa
commit
2eda00d9b9
14 changed files with 80 additions and 816 deletions
|
@ -58,8 +58,5 @@
|
||||||
"cypress": "13.4.0",
|
"cypress": "13.4.0",
|
||||||
"eslint": "8.52.0",
|
"eslint": "8.52.0",
|
||||||
"start-server-and-test": "2.0.1"
|
"start-server-and-test": "2.0.1"
|
||||||
},
|
|
||||||
"optionalDependencies": {
|
|
||||||
"@tensorflow/tfjs-core": "4.4.0"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -38,8 +38,6 @@
|
||||||
"@swc/core-win32-arm64-msvc": "1.3.56",
|
"@swc/core-win32-arm64-msvc": "1.3.56",
|
||||||
"@swc/core-win32-ia32-msvc": "1.3.56",
|
"@swc/core-win32-ia32-msvc": "1.3.56",
|
||||||
"@swc/core-win32-x64-msvc": "1.3.56",
|
"@swc/core-win32-x64-msvc": "1.3.56",
|
||||||
"@tensorflow/tfjs": "4.4.0",
|
|
||||||
"@tensorflow/tfjs-node": "4.4.0",
|
|
||||||
"bufferutil": "4.0.7",
|
"bufferutil": "4.0.7",
|
||||||
"slacc-android-arm-eabi": "0.0.10",
|
"slacc-android-arm-eabi": "0.0.10",
|
||||||
"slacc-android-arm64": "0.0.10",
|
"slacc-android-arm64": "0.0.10",
|
||||||
|
@ -129,7 +127,6 @@
|
||||||
"nested-property": "4.0.0",
|
"nested-property": "4.0.0",
|
||||||
"node-fetch": "3.3.2",
|
"node-fetch": "3.3.2",
|
||||||
"nodemailer": "6.9.7",
|
"nodemailer": "6.9.7",
|
||||||
"nsfwjs": "2.4.2",
|
|
||||||
"oauth": "0.10.0",
|
"oauth": "0.10.0",
|
||||||
"oauth2orize": "1.12.0",
|
"oauth2orize": "1.12.0",
|
||||||
"oauth2orize-pkce": "0.1.2",
|
"oauth2orize-pkce": "0.1.2",
|
||||||
|
|
|
@ -1,72 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and other misskey contributors
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import * as fs from 'node:fs';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
import { dirname } from 'node:path';
|
|
||||||
import { Injectable } from '@nestjs/common';
|
|
||||||
import * as nsfw from 'nsfwjs';
|
|
||||||
import si from 'systeminformation';
|
|
||||||
import { Mutex } from 'async-mutex';
|
|
||||||
import { bindThis } from '@/decorators.js';
|
|
||||||
|
|
||||||
const _filename = fileURLToPath(import.meta.url);
|
|
||||||
const _dirname = dirname(_filename);
|
|
||||||
|
|
||||||
const REQUIRED_CPU_FLAGS = ['avx2', 'fma'];
|
|
||||||
let isSupportedCpu: undefined | boolean = undefined;
|
|
||||||
|
|
||||||
@Injectable()
|
|
||||||
export class AiService {
|
|
||||||
private model: nsfw.NSFWJS;
|
|
||||||
private modelLoadMutex: Mutex = new Mutex();
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
@bindThis
|
|
||||||
public async detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
|
|
||||||
try {
|
|
||||||
if (isSupportedCpu === undefined) {
|
|
||||||
const cpuFlags = await this.getCpuFlags();
|
|
||||||
isSupportedCpu = REQUIRED_CPU_FLAGS.every(required => cpuFlags.includes(required));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isSupportedCpu) {
|
|
||||||
console.error('This CPU cannot use TensorFlow.');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const tf = await import('@tensorflow/tfjs-node');
|
|
||||||
|
|
||||||
if (this.model == null) {
|
|
||||||
await this.modelLoadMutex.runExclusive(async () => {
|
|
||||||
if (this.model == null) {
|
|
||||||
this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const buffer = await fs.promises.readFile(path);
|
|
||||||
const image = await tf.node.decodeImage(buffer, 3) as any;
|
|
||||||
try {
|
|
||||||
const predictions = await this.model.classify(image);
|
|
||||||
return predictions;
|
|
||||||
} finally {
|
|
||||||
image.dispose();
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@bindThis
|
|
||||||
private async getCpuFlags(): Promise<string[]> {
|
|
||||||
const str = await si.cpuFlags();
|
|
||||||
return str.split(/\s+/);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -6,7 +6,6 @@
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { AccountMoveService } from './AccountMoveService.js';
|
import { AccountMoveService } from './AccountMoveService.js';
|
||||||
import { AccountUpdateService } from './AccountUpdateService.js';
|
import { AccountUpdateService } from './AccountUpdateService.js';
|
||||||
import { AiService } from './AiService.js';
|
|
||||||
import { AnnouncementService } from './AnnouncementService.js';
|
import { AnnouncementService } from './AnnouncementService.js';
|
||||||
import { AntennaService } from './AntennaService.js';
|
import { AntennaService } from './AntennaService.js';
|
||||||
import { AppLockService } from './AppLockService.js';
|
import { AppLockService } from './AppLockService.js';
|
||||||
|
@ -138,7 +137,6 @@ import type { Provider } from '@nestjs/common';
|
||||||
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
|
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
|
||||||
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
|
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
|
||||||
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
|
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
|
||||||
const $AiService: Provider = { provide: 'AiService', useExisting: AiService };
|
|
||||||
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
|
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
|
||||||
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
|
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
|
||||||
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
|
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
|
||||||
|
@ -274,7 +272,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
||||||
LoggerService,
|
LoggerService,
|
||||||
AccountMoveService,
|
AccountMoveService,
|
||||||
AccountUpdateService,
|
AccountUpdateService,
|
||||||
AiService,
|
|
||||||
AnnouncementService,
|
AnnouncementService,
|
||||||
AntennaService,
|
AntennaService,
|
||||||
AppLockService,
|
AppLockService,
|
||||||
|
@ -403,7 +400,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
||||||
$LoggerService,
|
$LoggerService,
|
||||||
$AccountMoveService,
|
$AccountMoveService,
|
||||||
$AccountUpdateService,
|
$AccountUpdateService,
|
||||||
$AiService,
|
|
||||||
$AnnouncementService,
|
$AnnouncementService,
|
||||||
$AntennaService,
|
$AntennaService,
|
||||||
$AppLockService,
|
$AppLockService,
|
||||||
|
@ -533,7 +529,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
||||||
LoggerService,
|
LoggerService,
|
||||||
AccountMoveService,
|
AccountMoveService,
|
||||||
AccountUpdateService,
|
AccountUpdateService,
|
||||||
AiService,
|
|
||||||
AnnouncementService,
|
AnnouncementService,
|
||||||
AntennaService,
|
AntennaService,
|
||||||
AppLockService,
|
AppLockService,
|
||||||
|
@ -661,7 +656,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
||||||
$LoggerService,
|
$LoggerService,
|
||||||
$AccountMoveService,
|
$AccountMoveService,
|
||||||
$AccountUpdateService,
|
$AccountUpdateService,
|
||||||
$AiService,
|
|
||||||
$AnnouncementService,
|
$AnnouncementService,
|
||||||
$AntennaService,
|
$AntennaService,
|
||||||
$AppLockService,
|
$AppLockService,
|
||||||
|
|
|
@ -5,19 +5,13 @@
|
||||||
|
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
import * as crypto from 'node:crypto';
|
import * as crypto from 'node:crypto';
|
||||||
import { join } from 'node:path';
|
|
||||||
import * as stream from 'node:stream/promises';
|
import * as stream from 'node:stream/promises';
|
||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { FSWatcher } from 'chokidar';
|
|
||||||
import * as fileType from 'file-type';
|
import * as fileType from 'file-type';
|
||||||
import FFmpeg from 'fluent-ffmpeg';
|
|
||||||
import isSvg from 'is-svg';
|
import isSvg from 'is-svg';
|
||||||
import probeImageSize from 'probe-image-size';
|
import probeImageSize from 'probe-image-size';
|
||||||
import { type predictionType } from 'nsfwjs';
|
|
||||||
import sharp from 'sharp';
|
import sharp from 'sharp';
|
||||||
import { encode } from 'blurhash';
|
import { encode } from 'blurhash';
|
||||||
import { createTempDir } from '@/misc/create-temp.js';
|
|
||||||
import { AiService } from '@/core/AiService.js';
|
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
|
||||||
export type FileInfo = {
|
export type FileInfo = {
|
||||||
|
@ -49,7 +43,6 @@ const TYPE_SVG = {
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class FileInfoService {
|
export class FileInfoService {
|
||||||
constructor(
|
constructor(
|
||||||
private aiService: AiService,
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,7 +50,8 @@ export class FileInfoService {
|
||||||
* Get file information
|
* Get file information
|
||||||
*/
|
*/
|
||||||
@bindThis
|
@bindThis
|
||||||
public async getFileInfo(path: string, opts: {
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
|
public async getFileInfo(path: string, _opts: {
|
||||||
skipSensitiveDetection: boolean;
|
skipSensitiveDetection: boolean;
|
||||||
sensitiveThreshold?: number;
|
sensitiveThreshold?: number;
|
||||||
sensitiveThresholdForPorn?: number;
|
sensitiveThresholdForPorn?: number;
|
||||||
|
@ -128,22 +122,8 @@ export class FileInfoService {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let sensitive = false;
|
const sensitive = false;
|
||||||
let porn = false;
|
const porn = false;
|
||||||
|
|
||||||
if (!opts.skipSensitiveDetection) {
|
|
||||||
await this.detectSensitivity(
|
|
||||||
path,
|
|
||||||
type.mime,
|
|
||||||
opts.sensitiveThreshold ?? 0.5,
|
|
||||||
opts.sensitiveThresholdForPorn ?? 0.75,
|
|
||||||
opts.enableSensitiveMediaDetectionForVideos ?? false,
|
|
||||||
).then(value => {
|
|
||||||
[sensitive, porn] = value;
|
|
||||||
}, error => {
|
|
||||||
warnings.push(`detectSensitivity failed: ${error}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
size,
|
size,
|
||||||
|
@ -159,150 +139,6 @@ export class FileInfoService {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
|
||||||
private async detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
|
|
||||||
let sensitive = false;
|
|
||||||
let porn = false;
|
|
||||||
|
|
||||||
function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
|
|
||||||
let sensitive = false;
|
|
||||||
let porn = false;
|
|
||||||
|
|
||||||
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
|
||||||
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
|
||||||
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
|
||||||
|
|
||||||
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
|
|
||||||
|
|
||||||
return [sensitive, porn];
|
|
||||||
}
|
|
||||||
|
|
||||||
if ([
|
|
||||||
'image/jpeg',
|
|
||||||
'image/png',
|
|
||||||
'image/webp',
|
|
||||||
].includes(mime)) {
|
|
||||||
const result = await this.aiService.detectSensitive(source);
|
|
||||||
if (result) {
|
|
||||||
[sensitive, porn] = judgePrediction(result);
|
|
||||||
}
|
|
||||||
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
|
|
||||||
const [outDir, disposeOutDir] = await createTempDir();
|
|
||||||
try {
|
|
||||||
const command = FFmpeg()
|
|
||||||
.input(source)
|
|
||||||
.inputOptions([
|
|
||||||
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする(そうなるとは限らない)
|
|
||||||
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする(そうなるとは限らない)
|
|
||||||
])
|
|
||||||
.noAudio()
|
|
||||||
.videoFilters([
|
|
||||||
{
|
|
||||||
filter: 'select', // フレームのフィルタリング
|
|
||||||
options: {
|
|
||||||
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする(VP9 とかはデコードしてみないとわからないっぽい)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
filter: 'blackframe', // 暗いフレームの検出
|
|
||||||
options: {
|
|
||||||
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
filter: 'metadata',
|
|
||||||
options: {
|
|
||||||
mode: 'select', // フレーム選択モード
|
|
||||||
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率(前のフィルタからのメタデータを参照する)
|
|
||||||
value: '50',
|
|
||||||
function: 'less', // 50% 未満のフレームを選択する(50% 以上暗部があるフレームだと誤検知を招くかもしれないので)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
filter: 'scale',
|
|
||||||
options: {
|
|
||||||
w: 299,
|
|
||||||
h: 299,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
])
|
|
||||||
.format('image2')
|
|
||||||
.output(join(outDir, '%d.png'))
|
|
||||||
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
|
|
||||||
const results: ReturnType<typeof judgePrediction>[] = [];
|
|
||||||
let frameIndex = 0;
|
|
||||||
let targetIndex = 0;
|
|
||||||
let nextIndex = 1;
|
|
||||||
for await (const path of this.asyncIterateFrames(outDir, command)) {
|
|
||||||
try {
|
|
||||||
const index = frameIndex++;
|
|
||||||
if (index !== targetIndex) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
targetIndex = nextIndex;
|
|
||||||
nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
|
|
||||||
const result = await this.aiService.detectSensitive(path);
|
|
||||||
if (result) {
|
|
||||||
results.push(judgePrediction(result));
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
fs.promises.unlink(path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
|
|
||||||
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
|
|
||||||
} finally {
|
|
||||||
disposeOutDir();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return [sensitive, porn];
|
|
||||||
}
|
|
||||||
|
|
||||||
private async *asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
|
|
||||||
const watcher = new FSWatcher({
|
|
||||||
cwd,
|
|
||||||
disableGlobbing: true,
|
|
||||||
});
|
|
||||||
let finished = false;
|
|
||||||
command.once('end', () => {
|
|
||||||
finished = true;
|
|
||||||
watcher.close();
|
|
||||||
});
|
|
||||||
command.run();
|
|
||||||
for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
|
||||||
const current = `${i}.png`;
|
|
||||||
const next = `${i + 1}.png`;
|
|
||||||
const framePath = join(cwd, current);
|
|
||||||
if (await this.exists(join(cwd, next))) {
|
|
||||||
yield framePath;
|
|
||||||
} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
|
||||||
watcher.add(next);
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
watcher.on('add', function onAdd(path) {
|
|
||||||
if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
|
|
||||||
watcher.unwatch(current);
|
|
||||||
watcher.off('add', onAdd);
|
|
||||||
resolve();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
|
|
||||||
command.once('error', reject);
|
|
||||||
});
|
|
||||||
yield framePath;
|
|
||||||
} else if (await this.exists(framePath)) {
|
|
||||||
yield framePath;
|
|
||||||
} else {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@bindThis
|
|
||||||
private exists(path: string): Promise<boolean> {
|
|
||||||
return fs.promises.access(path).then(() => true, () => false);
|
|
||||||
}
|
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public fixMime(mime: string | fileType.MimeType): string {
|
public fixMime(mime: string | fileType.MimeType): string {
|
||||||
// see https://github.com/misskey-dev/misskey/pull/10686
|
// see https://github.com/misskey-dev/misskey/pull/10686
|
||||||
|
|
|
@ -14,7 +14,6 @@ import { describe, beforeAll, afterAll, test } from '@jest/globals';
|
||||||
import { GlobalModule } from '@/GlobalModule.js';
|
import { GlobalModule } from '@/GlobalModule.js';
|
||||||
import { FileInfoService } from '@/core/FileInfoService.js';
|
import { FileInfoService } from '@/core/FileInfoService.js';
|
||||||
//import { DI } from '@/di-symbols.js';
|
//import { DI } from '@/di-symbols.js';
|
||||||
import { AiService } from '@/core/AiService.js';
|
|
||||||
import type { TestingModule } from '@nestjs/testing';
|
import type { TestingModule } from '@nestjs/testing';
|
||||||
import type { MockFunctionMetadata } from 'jest-mock';
|
import type { MockFunctionMetadata } from 'jest-mock';
|
||||||
|
|
||||||
|
@ -34,14 +33,10 @@ describe('FileInfoService', () => {
|
||||||
GlobalModule,
|
GlobalModule,
|
||||||
],
|
],
|
||||||
providers: [
|
providers: [
|
||||||
AiService,
|
|
||||||
FileInfoService,
|
FileInfoService,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
.useMocker((token) => {
|
.useMocker((token) => {
|
||||||
//if (token === AiService) {
|
|
||||||
// return { };
|
|
||||||
//}
|
|
||||||
if (typeof token === 'function') {
|
if (typeof token === 'function') {
|
||||||
const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata<any, any>;
|
const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata<any, any>;
|
||||||
const Mock = moduleMocker.generateFromMetadata(mockMetadata);
|
const Mock = moduleMocker.generateFromMetadata(mockMetadata);
|
||||||
|
|
605
pnpm-lock.yaml
605
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue