Merge branch 'develop'

CHANGELOG.md

@@ -35,6 +35,19 @@ mongodb:
 8. master ブランチに戻す
 9. enjoy
 
+11.4.0 (2019/04/25)
+-------------------
+### Improvements
+* 検索でローカルの投稿のみに絞れるように
+* 検索で特定のインスタンスの投稿のみに絞れるように
+* 検索で特定のユーザーの投稿のみに絞れるように
+
+### Fixes
+* 投稿が増殖する問題を修正
+* ストリームで過去の投稿が流れてくる問題を修正
+* モバイル版のユーザーページで遷移してもユーザー名が変わらない問題を修正
+* お知らせを切り替えても内容が変わらない問題を修正
+
 11.3.1 (2019/04/24)
 -------------------
 ### Fixes

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "misskey",
 	"author": "syuilo <i@syuilo.com>",
-	"version": "11.3.1",
+	"version": "11.4.0",
 	"codename": "daybreak",
 	"repository": {
 		"type": "git",
@@ -23,6 +23,7 @@
 		"format": "gulp format"
 	},
 	"dependencies": {
+		"@elastic/elasticsearch": "7.0.0-rc.2",
 		"@fortawesome/fontawesome-svg-core": "1.2.15",
 		"@fortawesome/free-brands-svg-icons": "5.7.2",
 		"@fortawesome/free-regular-svg-icons": "5.7.2",
@@ -35,7 +36,6 @@
 		"@types/dateformat": "3.0.0",
 		"@types/deep-equal": "1.0.1",
 		"@types/double-ended-queue": "2.1.0",
-		"@types/elasticsearch": "5.0.32",
 		"@types/file-type": "10.9.1",
 		"@types/gulp": "4.0.6",
 		"@types/gulp-mocha": "0.0.32",
@@ -113,7 +113,6 @@
 		"deep-equal": "1.0.1",
 		"diskusage": "1.1.0",
 		"double-ended-queue": "2.1.0-0",
-		"elasticsearch": "15.4.1",
 		"emojilib": "2.4.0",
 		"eslint": "5.16.0",
 		"eslint-plugin-vue": "5.2.2",

src/client/app/common/scripts/gen-search-query.ts

@@ -0,0 +1,31 @@
+import parseAcct from '../../../../misc/acct/parse';
+import { host as localHost } from '../../config';
+
+export async function genSearchQuery(v: any, q: string) {
+	let host: string;
+	let userId: string;
+	if (q.split(' ').some(x => x.startsWith('@'))) {
+		for (const at of q.split(' ').filter(x => x.startsWith('@')).map(x => x.substr(1))) {
+			if (at.includes('.')) {
+				if (at === localHost || at === '.') {
+					host = null;
+				} else {
+					host = at;
+				}
+			} else {
+				const user = await v.$root.api('users/show', parseAcct(at)).catch(x => null);
+				if (user) {
+					userId = user.id;
+				} else {
+					// todo: show error
+				}
+			}
+		}
+
+	}
+	return {
+		query: q.split(' ').filter(x => !x.startsWith('/') && !x.startsWith('@')).join(' '),
+		host: host,
+		userId: userId
+	};
+}

src/client/app/common/scripts/search.ts

@@ -3,7 +3,7 @@ import { faHistory } from '@fortawesome/free-solid-svg-icons';
 export async function search(v: any, q: string) {
 	q = q.trim();
 
-	if (q.startsWith('@')) {
+	if (q.startsWith('@') && !q.includes(' ')) {
 		v.$router.push(`/${q}`);
 		return;
 	}

src/client/app/common/views/components/user-list.vue

@@ -60,9 +60,9 @@ export default Vue.extend({
 	},
 
 	methods: {
-		init() {
+		async init() {
 			this.fetching = true;
-			this.makePromise().then(x => {
+			await (this.makePromise()).then(x => {
 				if (Array.isArray(x)) {
 					this.us = x;
 				} else {
@@ -76,9 +76,9 @@ export default Vue.extend({
 			});
 		},
 
-		fetchMoreUsers() {
+		async fetchMoreUsers() {
 			this.fetchingMoreUsers = true;
-			this.makePromise(this.cursor).then(x => {
+			await (this.makePromise(this.cursor)).then(x => {
 				this.us = this.us.concat(x.users);
 				this.cursor = x.cursor;
 				this.fetchingMoreUsers = false;

src/client/app/common/views/deck/deck.notes.vue

@@ -110,11 +110,11 @@ export default Vue.extend({
 			this.init();
 		},
 
-		init() {
+		async init() {
 			this.queue = [];
 			this.notes = [];
 			this.fetching = true;
-			this.makePromise().then(x => {
+			await (this.makePromise()).then(x => {
 				if (Array.isArray(x)) {
 					this.notes = x;
 				} else {
@@ -129,10 +129,10 @@ export default Vue.extend({
 			});
 		},
 
-		fetchMore() {
+		async fetchMore() {
 			if (!this.more || this.moreFetching) return;
 			this.moreFetching = true;
-			this.makePromise(this.notes[this.notes.length - 1].id).then(x => {
+			await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => {
 				this.notes = this.notes.concat(x.notes);
 				this.more = x.more;
 				this.moreFetching = false;

src/client/app/common/views/deck/deck.search-column.vue

@@ -14,6 +14,7 @@
 import Vue from 'vue';
 import XColumn from './deck.column.vue';
 import XNotes from './deck.notes.vue';
+import { genSearchQuery } from '../../../common/scripts/gen-search-query';
 
 const limit = 20;
 
@@ -25,10 +26,10 @@ export default Vue.extend({
 
 	data() {
 		return {
-			makePromise: cursor => this.$root.api('notes/search', {
+			makePromise: async cursor => this.$root.api('notes/search', {
 				limit: limit + 1,
 				offset: cursor ? cursor : undefined,
-				query: this.q
+				...(await genSearchQuery(this, this.q))
 			}).then(notes => {
 				if (notes.length == limit + 1) {
 					notes.pop();

src/client/app/common/views/widgets/broadcast.vue

@@ -18,7 +18,7 @@
 			<p class="fetching" v-if="fetching">{{ $t('fetching') }}<mk-ellipsis/></p>
 			<h1 v-if="!fetching">{{ announcements.length == 0 ? $t('no-broadcasts') : announcements[i].title }}</h1>
 			<p v-if="!fetching">
-				<mfm v-if="announcements.length != 0" :text="announcements[i].text"/>
+				<mfm v-if="announcements.length != 0" :text="announcements[i].text" :key="i"/>
 				<img v-if="announcements.length != 0 && announcements[i].image" :src="announcements[i].image" alt="" style="display: block; max-height: 130px; max-width: 100%;"/>
 				<template v-if="announcements.length == 0">{{ $t('have-a-nice-day') }}</template>
 			</p>

src/client/app/desktop/views/components/notes.vue

@@ -105,9 +105,9 @@ export default Vue.extend({
 			this.init();
 		},
 
-		init() {
+		async init() {
 			this.fetching = true;
-			this.makePromise().then(x => {
+			await (this.makePromise()).then(x => {
 				if (Array.isArray(x)) {
 					this.notes = x;
 				} else {
@@ -122,7 +122,7 @@ export default Vue.extend({
 			});
 		},
 
-		fetchMore() {
+		async fetchMore() {
 			if (!this.more || this.moreFetching || this.notes.length === 0) return;
 			this.moreFetching = true;
 			this.makePromise(this.notes[this.notes.length - 1].id).then(x => {

src/client/app/desktop/views/home/search.vue

@@ -14,6 +14,7 @@
 import Vue from 'vue';
 import i18n from '../../../i18n';
 import Progress from '../../../common/scripts/loading';
+import { genSearchQuery } from '../../../common/scripts/gen-search-query';
 
 const limit = 20;
 
@@ -21,10 +22,10 @@ export default Vue.extend({
 	i18n: i18n('desktop/views/pages/search.vue'),
 	data() {
 		return {
-			makePromise: cursor => this.$root.api('notes/search', {
+			makePromise: async cursor => this.$root.api('notes/search', {
 				limit: limit + 1,
 				offset: cursor ? cursor : undefined,
-				query: this.q
+				...(await genSearchQuery(this, this.q))
 			}).then(notes => {
 				if (notes.length == limit + 1) {
 					notes.pop();

src/client/app/mobile/views/components/notes.vue

@@ -106,9 +106,9 @@ export default Vue.extend({
 			this.init();
 		},
 
-		init() {
+		async init() {
 			this.fetching = true;
-			this.makePromise().then(x => {
+			await (this.makePromise()).then(x => {
 				if (Array.isArray(x)) {
 					this.notes = x;
 				} else {
@@ -123,10 +123,10 @@ export default Vue.extend({
 			});
 		},
 
-		fetchMore() {
+		async fetchMore() {
 			if (!this.more || this.moreFetching || this.notes.length === 0) return;
 			this.moreFetching = true;
-			this.makePromise(this.notes[this.notes.length - 1].id).then(x => {
+			await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => {
 				this.notes = this.notes.concat(x.notes);
 				this.more = x.more;
 				this.moreFetching = false;

src/client/app/mobile/views/components/ui.nav.vue

@@ -43,7 +43,7 @@
 				<div class="announcements" v-if="announcements && announcements.length > 0">
 					<article v-for="announcement in announcements">
 						<span v-html="announcement.title" class="title"></span>
-						<mfm :text="announcement.text"/>
+						<div><mfm :text="announcement.text"/></div>
 						<img v-if="announcement.image" :src="announcement.image" alt="" style="display: block; max-height: 120px; max-width: 100%;"/>
 					</article>
 				</div>

src/client/app/mobile/views/pages/search.vue

@@ -12,6 +12,7 @@
 import Vue from 'vue';
 import i18n from '../../../i18n';
 import Progress from '../../../common/scripts/loading';
+import { genSearchQuery } from '../../../common/scripts/gen-search-query';
 
 const limit = 20;
 
@@ -19,10 +20,10 @@ export default Vue.extend({
 	i18n: i18n('mobile/views/pages/search.vue'),
 	data() {
 		return {
-			makePromise: cursor => this.$root.api('notes/search', {
+			makePromise: async cursor => this.$root.api('notes/search', {
 				limit: limit + 1,
 				untilId: cursor ? cursor : undefined,
-				query: this.q
+				...(await genSearchQuery(this, this.q))
 			}).then(notes => {
 				if (notes.length == limit + 1) {
 					notes.pop();

src/client/app/mobile/views/pages/user/index.vue

@@ -18,7 +18,7 @@
 				</div>
 				<div class="title">
 					<h1><mk-user-name :user="user" :key="user.id"/></h1>
-					<span class="username"><mk-acct :user="user" :detail="true" /></span>
+					<span class="username"><mk-acct :user="user" :detail="true" :key="user.id"/></span>
 					<span class="followed" v-if="user.isFollowed">{{ $t('follows-you') }}</span>
 				</div>
 				<div class="description">

src/db/elasticsearch.ts

@@ -1,41 +1,30 @@
-import * as elasticsearch from 'elasticsearch';
+import * as elasticsearch from '@elastic/elasticsearch';
 import config from '../config';
-import Logger from '../services/logger';
-
-const esLogger = new Logger('es');
 
 const index = {
 	settings: {
 		analysis: {
-			normalizer: {
-				lowercase_normalizer: {
-					type: 'custom',
-					filter: ['lowercase']
-				}
-			},
 			analyzer: {
-				bigram: {
+				ngram: {
-					tokenizer: 'bigram_tokenizer'
+					tokenizer: 'ngram'
-				}
-			},
-			tokenizer: {
-				bigram_tokenizer: {
-					type: 'nGram',
-					min_gram: 2,
-					max_gram: 2
 				}
 			}
 		}
 	},
 	mappings: {
-		note: {
+		properties: {
-			properties: {
+			text: {
-				text: {
+				type: 'text',
-					type: 'text',
+				index: true,
-					index: true,
+				analyzer: 'ngram',
-					analyzer: 'bigram',
+			},
-					normalizer: 'lowercase_normalizer'
+			userId: {
-				}
+				type: 'keyword',
+				index: true,
+			},
+			userHost: {
+				type: 'keyword',
+				index: true,
 			}
 		}
 	}
@@ -43,31 +32,20 @@ const index = {
 
 // Init ElasticSearch connection
 const client = config.elasticsearch ? new elasticsearch.Client({
-	host: `${config.elasticsearch.host}:${config.elasticsearch.port}`
+	node: `http://${config.elasticsearch.host}:${config.elasticsearch.port}`,
+	pingTimeout: 30000
 }) : null;
 
 if (client) {
-	// Send a HEAD request
-	client.ping({
-		// Ping usually has a 3000ms timeout
-		requestTimeout: 30000
-	}, error => {
-		if (error) {
-			esLogger.error('elasticsearch is down!');
-		} else {
-			esLogger.succ('elasticsearch is available!');
-		}
-	});
-
 	client.indices.exists({
-		index: 'misskey'
+		index: 'misskey_note'
 	}).then(exist => {
-		if (exist) return;
+		if (!exist.body) {
-
+			client.indices.create({
-		client.indices.create({
+				index: 'misskey_note',
-			index: 'misskey',
+				body: index
-			body: index
+			});
-		});
+		}
 	});
 }
 

src/remote/activitypub/models/note.ts

@@ -247,7 +247,7 @@ export async function resolveNote(value: string | IObject, resolver?: Resolver):
 	// リモートサーバーからフェッチしてきて登録
 	// ここでuriの代わりに添付されてきたNote Objectが指定されていると、サーバーフェッチを経ずにノートが生成されるが
 	// 添付されてきたNote Objectは偽装されている可能性があるため、常にuriを指定してサーバーフェッチを行う。
-	return await createNote(uri, resolver).catch(e => {
+	return await createNote(uri, resolver, true).catch(e => {
 		if (e.name === 'duplicated') {
 			return fetchNote(uri).then(note => {
 				if (note == null) {

src/server/api/endpoints/ap/show.ts

@@ -101,6 +101,32 @@ async function fetchAny(uri: string) {
 	// /@user のような正規id以外で取得できるURIが指定されていた場合、ここで初めて正規URIが確定する
 	// これはDBに存在する可能性があるため再度DB検索
 	if (uri !== object.id) {
+		if (object.id.startsWith(config.url + '/')) {
+			const parts = object.id.split('/');
+			const id = parts.pop();
+			const type = parts.pop();
+
+			if (type === 'notes') {
+				const note = await Notes.findOne(id);
+
+				if (note) {
+					return {
+						type: 'Note',
+						object: await Notes.pack(note, null, { detail: true })
+					};
+				}
+			} else if (type === 'users') {
+				const user = await Users.findOne(id);
+
+				if (user) {
+					return {
+						type: 'User',
+						object: await Users.pack(user, null, { detail: true })
+					};
+				}
+			}
+		}
+
 		const [user, note] = await Promise.all([
 			Users.findOne({ uri: object.id }),
 			Notes.findOne({ uri: object.id })
@@ -120,7 +146,7 @@ async function fetchAny(uri: string) {
 	}
 
 	if (['Note', 'Question', 'Article'].includes(object.type)) {
-		const note = await createNote(object.id);
+		const note = await createNote(object.id, undefined, true);
 		return {
 			type: 'Note',
 			object: await Notes.pack(note!, null, { detail: true })

src/server/api/endpoints/notes/search.ts

@@ -5,6 +5,7 @@ import { ApiError } from '../../error';
 import { Notes } from '../../../../models';
 import { In } from 'typeorm';
 import { types, bool } from '../../../../misc/schema';
+import { ID } from '../../../../misc/cafy-id';
 
 export const meta = {
 	desc: {
@@ -29,7 +30,17 @@ export const meta = {
 		offset: {
 			validator: $.optional.num.min(0),
 			default: 0
-		}
+		},
+
+		host: {
+			validator: $.optional.nullable.str,
+			default: undefined
+		},
+
+		userId: {
+			validator: $.optional.nullable.type(ID),
+			default: null
+		},
 	},
 
 	res: {
@@ -54,30 +65,51 @@ export const meta = {
 export default define(meta, async (ps, me) => {
 	if (es == null) throw new ApiError(meta.errors.searchingNotAvailable);
 
-	const response = await es.search({
+	const userQuery = ps.userId != null ? [{
-		index: 'misskey',
+		term: {
-		type: 'note',
+			userId: ps.userId
+		}
+	}] : [];
+
+	const hostQuery = ps.userId == null ?
+		ps.host === null ? [{
+			bool: {
+				must_not: {
+					exists: {
+						field: 'userHost'
+					}
+				}
+			}
+		}] : ps.host !== undefined ? [{
+			term: {
+				userHost: ps.host
+			}
+		}] : []
+	: [];
+
+	const result = await es.search({
+		index: 'misskey_note',
 		body: {
 			size: ps.limit!,
 			from: ps.offset,
 			query: {
-				simple_query_string: {
+				bool: {
-					fields: ['text'],
+					must: [{
-					query: ps.query,
+						simple_query_string: {
-					default_operator: 'and'
+							fields: ['text'],
+							query: ps.query.toLowerCase(),
+							default_operator: 'and'
+						},
+					}, ...hostQuery, ...userQuery]
 				}
 			},
-			sort: [
+			sort: [{
-				{ _doc: 'desc' }
+				_doc: 'desc'
-			]
+			}]
 		}
 	});
 
-	if (response.hits.total === 0) {
+	const hits = result.body.hits.hits.map((hit: any) => hit._id);
-		return [];
-	}
-
-	const hits = response.hits.hits.map((hit: any) => hit.id);
 
 	if (hits.length === 0) return [];
 

src/services/note/create.ts

@@ -106,8 +106,6 @@ type Option = {
 };
 
 export default async (user: User, data: Option, silent = false) => new Promise<Note>(async (res, rej) => {
-	const isFirstNote = user.notesCount === 0;
-
 	if (data.createdAt == null) data.createdAt = new Date();
 	if (data.visibility == null) data.visibility = 'public';
 	if (data.viaMobile == null) data.viaMobile = false;
@@ -195,8 +193,6 @@ export default async (user: User, data: Option, silent = false) => new Promise<N
 	// 統計を更新
 	notesChart.update(note, true);
 	perUserNotesChart.update(user, note, true);
-	// ローカルユーザーのチャートはタイムライン取得時に更新しているのでリモートユーザーの場合だけでよい
-	if (Users.isRemoteUser(user)) activeUsersChart.update(user);
 
 	// Register host
 	if (Users.isRemoteUser(user)) {
@@ -212,19 +208,6 @@ export default async (user: User, data: Option, silent = false) => new Promise<N
 	// Increment notes count (user)
 	incNotesCountOfUser(user);
 
-	// 未読通知を作成
-	if (data.visibility == 'specified') {
-		if (data.visibleUsers == null) throw new Error('invalid param');
-
-		for (const u of data.visibleUsers) {
-			insertNoteUnread(u, note, true);
-		}
-	} else {
-		for (const u of mentionedUsers) {
-			insertNoteUnread(u, note, false);
-		}
-	}
-
 	if (data.reply) {
 		saveReply(data.reply, note);
 	}
@@ -233,75 +216,91 @@ export default async (user: User, data: Option, silent = false) => new Promise<N
 		incRenoteCount(data.renote);
 	}
 
-	// Pack the note
-	const noteObj = await Notes.pack(note);
-
-	if (isFirstNote) {
-		(noteObj as any).isFirstNote = true;
-	}
-
-	publishNotesStream(noteObj);
-
-	const nm = new NotificationManager(user, note);
-	const nmRelatedPromises = [];
-
-	createMentionedEvents(mentionedUsers, note, nm);
-
-	const noteActivity = await renderNoteOrRenoteActivity(data, note);
-
-	if (Users.isLocalUser(user)) {
-		deliverNoteToMentionedRemoteUsers(mentionedUsers, user, noteActivity);
-	}
-
-	const profile = await UserProfiles.findOne(user.id).then(ensure);
-
-	// If has in reply to note
-	if (data.reply) {
-		// Fetch watchers
-		nmRelatedPromises.push(notifyToWatchersOfReplyee(data.reply, user, nm));
-
-		// この投稿をWatchする
-		if (Users.isLocalUser(user) && profile.autoWatch) {
-			watch(user.id, data.reply);
-		}
-
-		// 通知
-		if (data.reply.userHost === null) {
-			nm.push(data.reply.userId, 'reply');
-			publishMainStream(data.reply.userId, 'reply', noteObj);
-		}
-	}
-
-	// If it is renote
-	if (data.renote) {
-		const type = data.text ? 'quote' : 'renote';
-
-		// Notify
-		if (data.renote.userHost === null) {
-			nm.push(data.renote.userId, type);
-		}
-
-		// Fetch watchers
-		nmRelatedPromises.push(notifyToWatchersOfRenotee(data.renote, user, nm, type));
-
-		// この投稿をWatchする
-		if (Users.isLocalUser(user) && profile.autoWatch) {
-			watch(user.id, data.renote);
-		}
-
-		// Publish event
-		if ((user.id !== data.renote.userId) && data.renote.userHost === null) {
-			publishMainStream(data.renote.userId, 'renote', noteObj);
-		}
-	}
-
 	if (!silent) {
-		publish(user, note, data.reply, data.renote, noteActivity);
+		// ローカルユーザーのチャートはタイムライン取得時に更新しているのでリモートユーザーの場合だけでよい
-	}
+		if (Users.isRemoteUser(user)) activeUsersChart.update(user);
 
-	Promise.all(nmRelatedPromises).then(() => {
+		// 未読通知を作成
-		nm.deliver();
+		if (data.visibility == 'specified') {
-	});
+			if (data.visibleUsers == null) throw new Error('invalid param');
+
+			for (const u of data.visibleUsers) {
+				insertNoteUnread(u, note, true);
+			}
+		} else {
+			for (const u of mentionedUsers) {
+				insertNoteUnread(u, note, false);
+			}
+		}
+
+		// Pack the note
+		const noteObj = await Notes.pack(note);
+
+		if (user.notesCount === 0) {
+			(noteObj as any).isFirstNote = true;
+		}
+
+		publishNotesStream(noteObj);
+
+		const nm = new NotificationManager(user, note);
+		const nmRelatedPromises = [];
+
+		createMentionedEvents(mentionedUsers, note, nm);
+
+		const noteActivity = await renderNoteOrRenoteActivity(data, note);
+
+		if (Users.isLocalUser(user)) {
+			deliverNoteToMentionedRemoteUsers(mentionedUsers, user, noteActivity);
+		}
+
+		const profile = await UserProfiles.findOne(user.id).then(ensure);
+
+		// If has in reply to note
+		if (data.reply) {
+			// Fetch watchers
+			nmRelatedPromises.push(notifyToWatchersOfReplyee(data.reply, user, nm));
+
+			// この投稿をWatchする
+			if (Users.isLocalUser(user) && profile.autoWatch) {
+				watch(user.id, data.reply);
+			}
+
+			// 通知
+			if (data.reply.userHost === null) {
+				nm.push(data.reply.userId, 'reply');
+				publishMainStream(data.reply.userId, 'reply', noteObj);
+			}
+		}
+
+		// If it is renote
+		if (data.renote) {
+			const type = data.text ? 'quote' : 'renote';
+
+			// Notify
+			if (data.renote.userHost === null) {
+				nm.push(data.renote.userId, type);
+			}
+
+			// Fetch watchers
+			nmRelatedPromises.push(notifyToWatchersOfRenotee(data.renote, user, nm, type));
+
+			// この投稿をWatchする
+			if (Users.isLocalUser(user) && profile.autoWatch) {
+				watch(user.id, data.renote);
+			}
+
+			// Publish event
+			if ((user.id !== data.renote.userId) && data.renote.userHost === null) {
+				publishMainStream(data.renote.userId, 'renote', noteObj);
+			}
+		}
+
+		publish(user, note, data.reply, data.renote, noteActivity);
+
+		Promise.all(nmRelatedPromises).then(() => {
+			nm.deliver();
+		});
+	}
 
 	// Register to search database
 	index(note);
@@ -436,11 +435,12 @@ function index(note: Note) {
 	if (note.text == null || config.elasticsearch == null) return;
 
 	es!.index({
-		index: 'misskey',
+		index: 'misskey_note',
-		type: 'note',
 		id: note.id.toString(),
 		body: {
-			text: note.text
+			text: note.text.toLowerCase(),
+			userId: note.userId,
+			userHost: note.userHost
 		}
 	});
 }