From c6b0bf42a112f0d9afa8920d6497cc76205ecaf4 Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Wed, 6 Sep 2017 23:19:58 +0900
Subject: [PATCH] wip
---
locales/en.yml | 12 +++
locales/ja.yml | 12 +++
src/api/endpoints.ts | 4 +
src/api/endpoints/posts/categorize.ts | 52 +++++++++++++
src/tools/ai/categorizer.ts | 93 -----------------------
src/tools/ai/predict-all-post-category.ts | 57 ++++++++++++++
src/tools/ai/predict-user-interst.ts | 45 +++++++++++
src/web/app/common/tags/post-menu.tag | 23 ++++++
8 files changed, 205 insertions(+), 93 deletions(-)
create mode 100644 src/api/endpoints/posts/categorize.ts
delete mode 100644 src/tools/ai/categorizer.ts
create mode 100644 src/tools/ai/predict-all-post-category.ts
create mode 100644 src/tools/ai/predict-user-interst.ts
diff --git a/locales/en.yml b/locales/en.yml
index d40896212b..3b87ea758d 100644
--- a/locales/en.yml
+++ b/locales/en.yml
@@ -22,6 +22,14 @@ common:
confused: "Confused"
pudding: "Pudding"
+ post_categories:
+ music: "Music"
+ game: "Video Game"
+ anime: "Anime"
+ it: "IT"
+ gadgets: "Gadgets"
+ photography: "Photography"
+
input-message-here: "Enter message here"
send: "Send"
delete: "Delete"
@@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "Pin"
pinned: "Pinned"
+ select: "Select category"
+ categorize: "Accept"
+ categorized: "Category reported. Thank you!"
mk-reaction-picker:
choose-reaction: "Pick your reaction"
@@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter integration"
signin-history: "Sign in history"
api: "API"
+ link: "MisskeyLink"
settings: "Settings"
signout: "Sign out"
diff --git a/locales/ja.yml b/locales/ja.yml
index b8e5cff412..13d451b6d8 100644
--- a/locales/ja.yml
+++ b/locales/ja.yml
@@ -22,6 +22,14 @@ common:
confused: "こまこまのこまり"
pudding: "Pudding"
+ post_categories:
+ music: "音楽"
+ game: "ゲーム"
+ anime: "アニメ"
+ it: "IT"
+ gadgets: "ガジェット"
+ photography: "写真"
+
input-message-here: "ここにメッセージを入力"
send: "送信"
delete: "削除"
@@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "ピン留め"
pinned: "ピン留めしました"
+ select: "カテゴリを選択"
+ categorize: "決定"
+ categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。"
mk-reaction-picker:
choose-reaction: "リアクションを選択"
@@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter連携"
signin-history: "ログイン履歴"
api: "API"
+ link: "Misskeyリンク"
settings: "設定"
signout: "サインアウト"
diff --git a/src/api/endpoints.ts b/src/api/endpoints.ts
index e5be68c096..97b98895b8 100644
--- a/src/api/endpoints.ts
+++ b/src/api/endpoints.ts
@@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [
name: 'posts/trend',
withCredential: true
},
+ {
+ name: 'posts/categorize',
+ withCredential: true
+ },
{
name: 'posts/reactions',
withCredential: true
diff --git a/src/api/endpoints/posts/categorize.ts b/src/api/endpoints/posts/categorize.ts
new file mode 100644
index 0000000000..3530ba6bc4
--- /dev/null
+++ b/src/api/endpoints/posts/categorize.ts
@@ -0,0 +1,52 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import Post from '../../models/post';
+
+/**
+ * Categorize a post
+ *
+ * @param {any} params
+ * @param {any} user
+ * @return {Promise<any>}
+ */
+module.exports = (params, user) => new Promise(async (res, rej) => {
+ if (!user.is_pro) {
+ return rej('This endpoint is available only from a Pro account');
+ }
+
+ // Get 'post_id' parameter
+ const [postId, postIdErr] = $(params.post_id).id().$;
+ if (postIdErr) return rej('invalid post_id param');
+
+ // Get categorizee
+ const post = await Post.findOne({
+ _id: postId
+ });
+
+ if (post === null) {
+ return rej('post not found');
+ }
+
+ if (post.is_category_verified) {
+ return rej('This post already has the verified category');
+ }
+
+ // Get 'category' parameter
+ const [category, categoryErr] = $(params.category).string().or([
+ 'music', 'game', 'anime', 'it', 'gadgets', 'photography'
+ ]).$;
+ if (categoryErr) return rej('invalid category param');
+
+ // Set category
+ Post.update({ _id: post._id }, {
+ $set: {
+ category: category,
+ is_category_verified: true
+ }
+ });
+
+ // Send response
+ res();
+});
diff --git a/src/tools/ai/categorizer.ts b/src/tools/ai/categorizer.ts
deleted file mode 100644
index c13374161d..0000000000
--- a/src/tools/ai/categorizer.ts
+++ /dev/null
@@ -1,93 +0,0 @@
-import * as fs from 'fs';
-
-const bayes = require('./naive-bayes.js');
-const MeCab = require('mecab-async');
-import * as msgpack from 'msgpack-lite';
-
-import Post from '../../api/models/post';
-import config from '../../conf';
-
-/**
- * 投稿を学習したり与えられた投稿のカテゴリを予測します
- */
-export default class Categorizer {
- private classifier: any;
- private categorizerDbFilePath: string;
- private mecab: any;
-
- constructor() {
- this.categorizerDbFilePath = `${__dirname}/../../../data/category`;
-
- this.mecab = new MeCab();
- if (config.categorizer.mecab_command) this.mecab.command = config.categorizer.mecab_command;
-
- // BIND -----------------------------------
- this.tokenizer = this.tokenizer.bind(this);
- }
-
- private tokenizer(text: string) {
- return this.mecab.wakachiSync(text);
- }
-
- public async init() {
- try {
- const buffer = fs.readFileSync(this.categorizerDbFilePath);
- const db = msgpack.decode(buffer);
-
- this.classifier = bayes.import(db);
- this.classifier.tokenizer = this.tokenizer;
- } catch (e) {
- this.classifier = bayes({
- tokenizer: this.tokenizer
- });
-
- // 訓練データ
- const verifiedPosts = await Post.find({
- is_category_verified: true
- });
-
- // 学習
- verifiedPosts.forEach(post => {
- this.classifier.learn(post.text, post.category);
- });
-
- this.save();
- }
- }
-
- public async learn(id, category) {
- const post = await Post.findOne({ _id: id });
-
- Post.update({ _id: id }, {
- $set: {
- category: category,
- is_category_verified: true
- }
- });
-
- this.classifier.learn(post.text, category);
-
- this.save();
- }
-
- public async categorize(id) {
- const post = await Post.findOne({ _id: id });
-
- const category = this.classifier.categorize(post.text);
-
- Post.update({ _id: id }, {
- $set: {
- category: category
- }
- });
- }
-
- public async test(text) {
- return this.classifier.categorize(text);
- }
-
- private save() {
- const buffer = msgpack.encode(this.classifier.export());
- fs.writeFileSync(this.categorizerDbFilePath, buffer);
- }
-}
diff --git a/src/tools/ai/predict-all-post-category.ts b/src/tools/ai/predict-all-post-category.ts
new file mode 100644
index 0000000000..87e198b39b
--- /dev/null
+++ b/src/tools/ai/predict-all-post-category.ts
@@ -0,0 +1,57 @@
+const bayes = require('./naive-bayes.js');
+const MeCab = require('mecab-async');
+
+import Post from '../../api/models/post';
+import config from '../../conf';
+
+const classifier = bayes({
+ tokenizer: this.tokenizer
+});
+
+const mecab = new MeCab();
+if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
+
+// 訓練データ取得
+Post.find({
+ is_category_verified: true
+}, {
+ fields: {
+ _id: false,
+ text: true,
+ category: true
+ }
+}).then(verifiedPosts => {
+ // 学習
+ verifiedPosts.forEach(post => {
+ classifier.learn(post.text, post.category);
+ });
+
+ // 全ての(人間によって証明されていない)投稿を取得
+ Post.find({
+ text: {
+ $exists: true
+ },
+ is_category_verified: {
+ $ne: true
+ }
+ }, {
+ sort: {
+ _id: -1
+ },
+ fields: {
+ _id: true,
+ text: true
+ }
+ }).then(posts => {
+ posts.forEach(post => {
+ console.log(`predicting... ${post._id}`);
+ const category = classifier.categorize(post.text);
+
+ Post.update({ _id: post._id }, {
+ $set: {
+ category: category
+ }
+ });
+ });
+ });
+});
diff --git a/src/tools/ai/predict-user-interst.ts b/src/tools/ai/predict-user-interst.ts
new file mode 100644
index 0000000000..99bdfa4206
--- /dev/null
+++ b/src/tools/ai/predict-user-interst.ts
@@ -0,0 +1,45 @@
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+
+export async function predictOne(id) {
+ console.log(`predict interest of ${id} ...`);
+
+ // TODO: repostなども含める
+ const recentPosts = await Post.find({
+ user_id: id,
+ category: {
+ $exists: true
+ }
+ }, {
+ sort: {
+ _id: -1
+ },
+ limit: 1000,
+ fields: {
+ _id: false,
+ category: true
+ }
+ });
+
+ const categories = {};
+
+ recentPosts.forEach(post => {
+ if (categories[post.category]) {
+ categories[post.category]++;
+ } else {
+ categories[post.category] = 1;
+ }
+ });
+}
+
+export async function predictAll() {
+ const allUsers = await User.find({}, {
+ fields: {
+ _id: true
+ }
+ });
+
+ allUsers.forEach(user => {
+ predictOne(user._id);
+ });
+}
diff --git a/src/web/app/common/tags/post-menu.tag b/src/web/app/common/tags/post-menu.tag
index 33895212bc..be4468a214 100644
--- a/src/web/app/common/tags/post-menu.tag
+++ b/src/web/app/common/tags/post-menu.tag
@@ -2,6 +2,18 @@
<div class="backdrop" ref="backdrop" onclick={ close }></div>
<div class="popover { compact: opts.compact }" ref="popover">
<button if={ post.user_id === I.id } onclick={ pin }>%i18n:common.tags.mk-post-menu.pin%</button>
+ <div if={ I.is_pro && !post.is_category_verified }>
+ <select ref="categorySelect">
+ <option value="">%i18n:common.tags.mk-post-menu.select%</option>
+ <option value="music">%i18n:common.post_categories.music%</option>
+ <option value="game">%i18n:common.post_categories.game%</option>
+ <option value="anime">%i18n:common.post_categories.anime%</option>
+ <option value="it">%i18n:common.post_categories.it%</option>
+ <option value="gadgets">%i18n:common.post_categories.gadgets%</option>
+ <option value="photography">%i18n:common.post_categories.photography%</option>
+ </select>
+ <button onclick={ categorize }>%i18n:common.tags.mk-post-menu.categorize%</button>
+ </div>
</div>
<style>
$border-color = rgba(27, 31, 35, 0.15)
@@ -111,6 +123,17 @@
});
};
+ this.categorize = () => {
+ const category = this.refs.categorySelect.options[this.refs.categorySelect.selectedIndex].value;
+ this.api('posts/categorize', {
+ post_id: this.post.id,
+ category: category
+ }).then(() => {
+ if (this.opts.cb) this.opts.cb('categorized', '%i18n:common.tags.mk-post-menu.categorized%');
+ this.unmount();
+ });
+ };
+
this.close = () => {
this.refs.backdrop.style.pointerEvents = 'none';
anime({