From 0e8a4f36a24826909a2c0e864c4c131b56ea0e8a Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 20:55:51 +0900
Subject: [PATCH 01/29] start to improve serializers
From 78487934c7e55b36b07f30b73127577ddec31f32 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 21:11:16 +0900
Subject: [PATCH 02/29] selializers - posts: unneed async-await
Promise.all resolves all Promise, and selializeDriveFile returns Promise.
---
src/api/serializers/post.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index 7c3690ef79..b2c54e9df8 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -84,8 +84,8 @@ const self = (
// Populate media
if (_post.media_ids) {
- _post.media = await Promise.all(_post.media_ids.map(async fileId =>
- await serializeDriveFile(fileId)
+ _post.media = await Promise.all(_post.media_ids.map(fileId =>
+ serializeDriveFile(fileId)
));
}
From 11190f56ad85a12faaa3653f8743dd75948ff11e Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:13:28 +0900
Subject: [PATCH 03/29] serializers/post - run promises in parallel
now w/ opts.detail, returns my_reaction field as 'null' w/ no reaction
(before: field appears w/ some reaction)
---
package.json | 1 +
src/api/serializers/post.ts | 122 ++++++++++++++++++++----------------
2 files changed, 70 insertions(+), 53 deletions(-)
diff --git a/package.json b/package.json
index 051eb1cb83..1e6e8d8136 100644
--- a/package.json
+++ b/package.json
@@ -95,6 +95,7 @@
"webpack": "3.8.1"
},
"dependencies": {
+ "@prezzemolo/rap": "^0.1.0",
"accesses": "2.5.0",
"animejs": "2.2.0",
"autwh": "0.0.1",
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index b2c54e9df8..352932acff 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -12,6 +12,7 @@ import serializeChannel from './channel';
import serializeUser from './user';
import serializeDriveFile from './drive-file';
import parse from '../common/text';
+import rap from '@prezzemolo/rap'
/**
* Serialize a post
@@ -70,21 +71,21 @@ const self = (
}
// Populate user
- _post.user = await serializeUser(_post.user_id, meId);
+ _post.user = serializeUser(_post.user_id, meId);
// Populate app
if (_post.app_id) {
- _post.app = await serializeApp(_post.app_id);
+ _post.app = serializeApp(_post.app_id);
}
// Populate channel
if (_post.channel_id) {
- _post.channel = await serializeChannel(_post.channel_id);
+ _post.channel = serializeChannel(_post.channel_id);
}
// Populate media
if (_post.media_ids) {
- _post.media = await Promise.all(_post.media_ids.map(fileId =>
+ _post.media = Promise.all(_post.media_ids.map(fileId =>
serializeDriveFile(fileId)
));
}
@@ -92,82 +93,97 @@ const self = (
// When requested a detailed post data
if (opts.detail) {
// Get previous post info
- const prev = await Post.findOne({
- user_id: _post.user_id,
- _id: {
- $lt: id
- }
- }, {
- fields: {
- _id: true
- },
- sort: {
- _id: -1
- }
- });
- _post.prev = prev ? prev._id : null;
+ _post.prev = (async () => {
+ const prev = Post.findOne({
+ user_id: _post.user_id,
+ _id: {
+ $lt: id
+ }
+ }, {
+ fields: {
+ _id: true
+ },
+ sort: {
+ _id: -1
+ }
+ });
+ return prev ? prev._id : null;
+ })()
// Get next post info
- const next = await Post.findOne({
- user_id: _post.user_id,
- _id: {
- $gt: id
- }
- }, {
- fields: {
- _id: true
- },
- sort: {
- _id: 1
- }
- });
- _post.next = next ? next._id : null;
+ _post.next = (async () => {
+ const next = await Post.findOne({
+ user_id: _post.user_id,
+ _id: {
+ $gt: id
+ }
+ }, {
+ fields: {
+ _id: true
+ },
+ sort: {
+ _id: 1
+ }
+ });
+ return next ? next._id : null;
+ })()
if (_post.reply_id) {
// Populate reply to post
- _post.reply = await self(_post.reply_id, meId, {
+ _post.reply = self(_post.reply_id, meId, {
detail: false
});
}
if (_post.repost_id) {
// Populate repost
- _post.repost = await self(_post.repost_id, meId, {
+ _post.repost = self(_post.repost_id, meId, {
detail: _post.text == null
});
}
// Poll
if (meId && _post.poll) {
- const vote = await Vote
- .findOne({
- user_id: meId,
- post_id: id
- });
+ _post.poll = (async (poll) => {
+ const vote = await Vote
+ .findOne({
+ user_id: meId,
+ post_id: id
+ });
- if (vote != null) {
- const myChoice = _post.poll.choices
- .filter(c => c.id == vote.choice)[0];
+ if (vote != null) {
+ const myChoice = poll.choices
+ .filter(c => c.id == vote.choice)[0];
- myChoice.is_voted = true;
- }
+ myChoice.is_voted = true;
+ }
+
+ return poll
+ })(_post.poll)
}
// Fetch my reaction
if (meId) {
- const reaction = await Reaction
- .findOne({
- user_id: meId,
- post_id: id,
- deleted_at: { $exists: false }
- });
+ _post.my_reaction = (async () => {
+ const reaction = await Reaction
+ .findOne({
+ user_id: meId,
+ post_id: id,
+ deleted_at: { $exists: false }
+ });
- if (reaction) {
- _post.my_reaction = reaction.reaction;
- }
+ if (reaction) {
+ return reaction.reaction;
+ }
+
+ return null
+ })();
}
}
+ // resolve promises in _post object
+ _post = await rap(_post)
+
resolve(_post);
});
From 5aa5e5cc7074003cec3417636ea1972b6d88150d Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:22:49 +0900
Subject: [PATCH 04/29] serializers - user: run promises in parallel as
possible
---
src/api/serializers/post.ts | 2 +-
src/api/serializers/user.ts | 40 +++++++++++++++++++++----------------
2 files changed, 24 insertions(+), 18 deletions(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index 352932acff..99e9bb667c 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -12,7 +12,7 @@ import serializeChannel from './channel';
import serializeUser from './user';
import serializeDriveFile from './drive-file';
import parse from '../common/text';
-import rap from '@prezzemolo/rap'
+import rap from '@prezzemolo/rap';
/**
* Serialize a post
diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts
index 3deff2d003..3527921ded 100644
--- a/src/api/serializers/user.ts
+++ b/src/api/serializers/user.ts
@@ -8,6 +8,7 @@ import serializePost from './post';
import Following from '../models/following';
import getFriends from '../common/get-friends';
import config from '../../conf';
+import rap from '@prezzemolo/rap';
/**
* Serialize a user
@@ -104,26 +105,30 @@ export default (
if (meId && !meId.equals(_user.id)) {
// If the user is following
- const follow = await Following.findOne({
- follower_id: meId,
- followee_id: _user.id,
- deleted_at: { $exists: false }
- });
- _user.is_following = follow !== null;
+ _user.is_following = (async () => {
+ const follow = await Following.findOne({
+ follower_id: meId,
+ followee_id: _user.id,
+ deleted_at: { $exists: false }
+ });
+ return follow !== null;
+ })()
// If the user is followed
- const follow2 = await Following.findOne({
- follower_id: _user.id,
- followee_id: meId,
- deleted_at: { $exists: false }
- });
- _user.is_followed = follow2 !== null;
+ _user.is_followed = (async () => {
+ const follow2 = await Following.findOne({
+ follower_id: _user.id,
+ followee_id: meId,
+ deleted_at: { $exists: false }
+ });
+ return follow2 !== null;
+ })()
}
if (opts.detail) {
if (_user.pinned_post_id) {
// Populate pinned post
- _user.pinned_post = await serializePost(_user.pinned_post_id, meId, {
+ _user.pinned_post = serializePost(_user.pinned_post_id, meId, {
detail: true
});
}
@@ -132,23 +137,24 @@ export default (
const myFollowingIds = await getFriends(meId);
// Get following you know count
- const followingYouKnowCount = await Following.count({
+ _user.following_you_know_count = Following.count({
followee_id: { $in: myFollowingIds },
follower_id: _user.id,
deleted_at: { $exists: false }
});
- _user.following_you_know_count = followingYouKnowCount;
// Get followers you know count
- const followersYouKnowCount = await Following.count({
+ _user.followers_you_know_count = Following.count({
followee_id: _user.id,
follower_id: { $in: myFollowingIds },
deleted_at: { $exists: false }
});
- _user.followers_you_know_count = followersYouKnowCount;
}
}
+ // resolve promises in _user object
+ _user = await rap(_user)
+
resolve(_user);
});
/*
From 7cd6b1c666605c7a256e4a8dd8db5edeb02da6db Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:26:16 +0900
Subject: [PATCH 05/29] follow lint
---
src/api/serializers/post.ts | 12 ++++++------
src/api/serializers/user.ts | 6 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index 99e9bb667c..e1ab784359 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -108,7 +108,7 @@ const self = (
}
});
return prev ? prev._id : null;
- })()
+ })();
// Get next post info
_post.next = (async () => {
@@ -126,7 +126,7 @@ const self = (
}
});
return next ? next._id : null;
- })()
+ })();
if (_post.reply_id) {
// Populate reply to post
@@ -158,8 +158,8 @@ const self = (
myChoice.is_voted = true;
}
- return poll
- })(_post.poll)
+ return poll;
+ })(_post.poll);
}
// Fetch my reaction
@@ -176,13 +176,13 @@ const self = (
return reaction.reaction;
}
- return null
+ return null;
})();
}
}
// resolve promises in _post object
- _post = await rap(_post)
+ _post = await rap(_post);
resolve(_post);
});
diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts
index 3527921ded..d00f073897 100644
--- a/src/api/serializers/user.ts
+++ b/src/api/serializers/user.ts
@@ -112,7 +112,7 @@ export default (
deleted_at: { $exists: false }
});
return follow !== null;
- })()
+ })();
// If the user is followed
_user.is_followed = (async () => {
@@ -122,7 +122,7 @@ export default (
deleted_at: { $exists: false }
});
return follow2 !== null;
- })()
+ })();
}
if (opts.detail) {
@@ -153,7 +153,7 @@ export default (
}
// resolve promises in _user object
- _user = await rap(_user)
+ _user = await rap(_user);
resolve(_user);
});
From 09baf205ead75eab3eaf0f3de82215665c2a3e73 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:29:58 +0900
Subject: [PATCH 06/29] remove ^ from @prezzemolo/rap dependency
---
package.json | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/package.json b/package.json
index 1e6e8d8136..c3a093420c 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,7 @@
"clean": "gulp clean",
"cleanall": "gulp cleanall",
"lint": "gulp lint",
- "test": "gulp test"
+ "test": "gulp test"
},
"devDependencies": {
"@types/bcryptjs": "2.4.0",
@@ -95,7 +95,7 @@
"webpack": "3.8.1"
},
"dependencies": {
- "@prezzemolo/rap": "^0.1.0",
+ "@prezzemolo/rap": "0.1.0",
"accesses": "2.5.0",
"animejs": "2.2.0",
"autwh": "0.0.1",
From 327d2705b4a3dad6ef8a8dfa8165c25a3a40d109 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:37:00 +0900
Subject: [PATCH 07/29] update @prezzemolo/rap to 0.1.1
---
package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/package.json b/package.json
index c3a093420c..27e292cc1a 100644
--- a/package.json
+++ b/package.json
@@ -95,7 +95,7 @@
"webpack": "3.8.1"
},
"dependencies": {
- "@prezzemolo/rap": "0.1.0",
+ "@prezzemolo/rap": "0.1.1",
"accesses": "2.5.0",
"animejs": "2.2.0",
"autwh": "0.0.1",
From ac2a0f46cd9ee877adda57bb939a1b31f7109911 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Sun, 5 Nov 2017 22:47:04 +0900
Subject: [PATCH 08/29] update @prezzemolo/rap to 0.1.2
---
package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/package.json b/package.json
index 27e292cc1a..6ea91a7f53 100644
--- a/package.json
+++ b/package.json
@@ -95,7 +95,7 @@
"webpack": "3.8.1"
},
"dependencies": {
- "@prezzemolo/rap": "0.1.1",
+ "@prezzemolo/rap": "0.1.2",
"accesses": "2.5.0",
"animejs": "2.2.0",
"autwh": "0.0.1",
From 7e81e0db6ac1289ae9504f7e3da5db6e56f41a51 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 14:37:00 +0900
Subject: [PATCH 09/29] support GridFS
---
src/api/common/add-file-to-drive.ts | 37 ++++++++++++++++++-----------
src/api/models/drive-file.ts | 15 ++++++++++--
src/db/mongodb.ts | 35 +++++++++++++++++++++++----
3 files changed, 67 insertions(+), 20 deletions(-)
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index 714eeb520d..f48f0cbcf5 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -4,14 +4,27 @@ import * as gm from 'gm';
import * as debug from 'debug';
import fileType = require('file-type');
import prominence = require('prominence');
-import DriveFile from '../models/drive-file';
+import DriveFile, { getGridFSBucket } from '../models/drive-file';
import DriveFolder from '../models/drive-folder';
import serialize from '../serializers/drive-file';
import event from '../event';
import config from '../../conf';
+import { Duplex } from 'stream';
const log = debug('misskey:register-drive-file');
+const addToGridFS = (name, binary, metadata): Promise<any> => new Promise(async (resolve, reject) => {
+ const dataStream = new Duplex()
+ dataStream.push(binary)
+ dataStream.push(null)
+
+ const bucket = await getGridFSBucket()
+ const writeStream = bucket.openUploadStream(name, { metadata })
+ writeStream.once('finish', (doc) => { resolve(doc) })
+ writeStream.on('error', reject)
+ dataStream.pipe(writeStream)
+})
+
/**
* Add file to drive
*
@@ -58,7 +71,7 @@ export default (
// Generate hash
const hash = crypto
- .createHash('sha256')
+ .createHash('md5')
.update(data)
.digest('hex') as string;
@@ -67,8 +80,10 @@ export default (
if (!force) {
// Check if there is a file with the same hash
const much = await DriveFile.findOne({
- user_id: user._id,
- hash: hash
+ md5: hash,
+ metadata: {
+ user_id: user._id
+ }
});
if (much !== null) {
@@ -82,13 +97,13 @@ export default (
// Calculate drive usage
const usage = ((await DriveFile
.aggregate([
- { $match: { user_id: user._id } },
+ { $match: { metadata: { user_id: user._id } } },
{ $project: {
- datasize: true
+ length: true
}},
{ $group: {
_id: null,
- usage: { $sum: '$datasize' }
+ usage: { $sum: '$length' }
}}
]))[0] || {
usage: 0
@@ -131,21 +146,15 @@ export default (
}
// Create DriveFile document
- const file = await DriveFile.insert({
- created_at: new Date(),
+ const file = await addToGridFS(`${user._id}/${name}`, data, {
user_id: user._id,
folder_id: folder !== null ? folder._id : null,
- data: data,
- datasize: size,
type: mime,
name: name,
comment: comment,
- hash: hash,
properties: properties
});
- delete file.data;
-
log(`drive file has been created ${file._id}`);
resolve(file);
diff --git a/src/api/models/drive-file.ts b/src/api/models/drive-file.ts
index 8d158cf563..79a87f6572 100644
--- a/src/api/models/drive-file.ts
+++ b/src/api/models/drive-file.ts
@@ -1,11 +1,22 @@
-import db from '../../db/mongodb';
+import * as mongodb from 'mongodb';
+import monkDb, { nativeDbConn } from '../../db/mongodb';
-const collection = db.get('drive_files');
+const collection = monkDb.get('drive_files.files');
(collection as any).createIndex('hash'); // fuck type definition
export default collection as any; // fuck type definition
+const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
+ const db = await nativeDbConn()
+ const bucket = new mongodb.GridFSBucket(db, {
+ bucketName: 'drive_files'
+ })
+ return bucket
+}
+
+export { getGridFSBucket }
+
export function validateFileName(name: string): boolean {
return (
(name.trim().length > 0) &&
diff --git a/src/db/mongodb.ts b/src/db/mongodb.ts
index 6ee7f4534f..75f1a1d3c6 100644
--- a/src/db/mongodb.ts
+++ b/src/db/mongodb.ts
@@ -1,11 +1,38 @@
-import * as mongo from 'monk';
-
import config from '../conf';
const uri = config.mongodb.user && config.mongodb.pass
- ? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`
- : `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
+? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`
+: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
+
+/**
+ * monk
+ */
+import * as mongo from 'monk';
const db = mongo(uri);
export default db;
+
+/**
+ * MongoDB native module (officialy)
+ */
+import * as mongodb from 'mongodb'
+
+let mdb: mongodb.Db;
+
+const nativeDbConn = async (): Promise<mongodb.Db> => {
+ if (mdb) return mdb;
+
+ const db = await ((): Promise<mongodb.Db> => new Promise((resolve, reject) => {
+ mongodb.MongoClient.connect(uri, (e, db) => {
+ if (e) return reject(e)
+ resolve(db)
+ })
+ }))()
+
+ mdb = db
+
+ return db
+}
+
+export { nativeDbConn }
From 18b1ef29adc6166c2b1a327b378c3e159a18b80c Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:18:45 +0900
Subject: [PATCH 10/29] migration to GridFS's DriveFile
---
src/api/common/add-file-to-drive.ts | 1 +
src/api/endpoints/drive.ts | 6 ++--
src/api/endpoints/drive/files.ts | 9 +++--
src/api/endpoints/drive/files/find.ts | 10 +++---
src/api/endpoints/drive/files/show.ts | 6 ++--
src/api/endpoints/drive/files/update.ts | 31 +++++++++--------
.../endpoints/messaging/messages/create.ts | 6 ++--
src/api/endpoints/posts/create.ts | 6 ++--
src/api/endpoints/posts/timeline.ts | 24 +++++++-------
src/api/serializers/drive-file.ts | 33 ++++++++-----------
src/api/serializers/drive-folder.ts | 4 ++-
11 files changed, 66 insertions(+), 70 deletions(-)
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index f48f0cbcf5..376c470e93 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -154,6 +154,7 @@ export default (
comment: comment,
properties: properties
});
+ console.dir(file)
log(`drive file has been created ${file._id}`);
diff --git a/src/api/endpoints/drive.ts b/src/api/endpoints/drive.ts
index 41ad6301d7..b9c4e3e506 100644
--- a/src/api/endpoints/drive.ts
+++ b/src/api/endpoints/drive.ts
@@ -14,16 +14,16 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Calculate drive usage
const usage = ((await DriveFile
.aggregate([
- { $match: { user_id: user._id } },
+ { $match: { metadata: { user_id: user._id } } },
{
$project: {
- datasize: true
+ length: true
}
},
{
$group: {
_id: null,
- usage: { $sum: '$datasize' }
+ usage: { $sum: '$length' }
}
}
]))[0] || {
diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts
index a68ae34817..eb0bfe6ba5 100644
--- a/src/api/endpoints/drive/files.ts
+++ b/src/api/endpoints/drive/files.ts
@@ -40,8 +40,10 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
_id: -1
};
const query = {
- user_id: user._id,
- folder_id: folderId
+ metadata: {
+ user_id: user._id,
+ folder_id: folderId
+ }
} as any;
if (sinceId) {
sort._id = 1;
@@ -57,9 +59,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
// Issue query
const files = await DriveFile
.find(query, {
- fields: {
- data: false
- },
limit: limit,
sort: sort
});
diff --git a/src/api/endpoints/drive/files/find.ts b/src/api/endpoints/drive/files/find.ts
index cd0b33f2ca..255faf94ec 100644
--- a/src/api/endpoints/drive/files/find.ts
+++ b/src/api/endpoints/drive/files/find.ts
@@ -24,12 +24,10 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Issue query
const files = await DriveFile
.find({
- name: name,
- user_id: user._id,
- folder_id: folderId
- }, {
- fields: {
- data: false
+ metadata: {
+ name: name,
+ user_id: user._id,
+ folder_id: folderId
}
});
diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts
index 8dbc297e4f..9135a04c57 100644
--- a/src/api/endpoints/drive/files/show.ts
+++ b/src/api/endpoints/drive/files/show.ts
@@ -21,10 +21,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
const file = await DriveFile
.findOne({
_id: fileId,
- user_id: user._id
- }, {
- fields: {
- data: false
+ metadata: {
+ user_id: user._id
}
});
diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts
index 1cfbdd8f0b..c4d2673688 100644
--- a/src/api/endpoints/drive/files/update.ts
+++ b/src/api/endpoints/drive/files/update.ts
@@ -20,25 +20,29 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
const [fileId, fileIdErr] = $(params.file_id).id().$;
if (fileIdErr) return rej('invalid file_id param');
+ console.dir(user)
+
// Fetch file
const file = await DriveFile
.findOne({
_id: fileId,
- user_id: user._id
- }, {
- fields: {
- data: false
+ metadata: {
+ user_id: user._id
}
});
+ console.dir(file)
+
if (file === null) {
return rej('file-not-found');
}
+ const updateQuery: any = {}
+
// Get 'name' parameter
const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
if (nameErr) return rej('invalid name param');
- if (name) file.name = name;
+ if (name) updateQuery.name = name;
// Get 'folder_id' parameter
const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
@@ -46,7 +50,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (folderId !== undefined) {
if (folderId === null) {
- file.folder_id = null;
+ updateQuery.folder_id = null;
} else {
// Fetch folder
const folder = await DriveFolder
@@ -59,19 +63,20 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('folder-not-found');
}
- file.folder_id = folder._id;
+ updateQuery.folder_id = folder._id;
}
}
- DriveFile.update(file._id, {
- $set: {
- name: file.name,
- folder_id: file.folder_id
- }
+ const updated = await DriveFile.update(file._id, {
+ $set: { metadata: updateQuery }
});
+ console.dir(updated)
+
// Serialize
- const fileObj = await serialize(file);
+ const fileObj = await serialize(updated);
+
+ console.dir(fileObj)
// Response
res(fileObj);
diff --git a/src/api/endpoints/messaging/messages/create.ts b/src/api/endpoints/messaging/messages/create.ts
index 8af55d850c..1d186268fb 100644
--- a/src/api/endpoints/messaging/messages/create.ts
+++ b/src/api/endpoints/messaging/messages/create.ts
@@ -54,9 +54,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (fileId !== undefined) {
file = await DriveFile.findOne({
_id: fileId,
- user_id: user._id
- }, {
- data: false
+ metadata: {
+ user_id: user._id
+ }
});
if (file === null) {
diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts
index f982b9ee93..1507639776 100644
--- a/src/api/endpoints/posts/create.ts
+++ b/src/api/endpoints/posts/create.ts
@@ -44,9 +44,9 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
// SELECT _id
const entity = await DriveFile.findOne({
_id: mediaId,
- user_id: user._id
- }, {
- _id: true
+ metadata: {
+ user_id: user._id
+ }
});
if (entity === null) {
diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts
index aa5aff5ba5..496de62b69 100644
--- a/src/api/endpoints/posts/timeline.ts
+++ b/src/api/endpoints/posts/timeline.ts
@@ -2,6 +2,7 @@
* Module dependencies
*/
import $ from 'cafy';
+import rap from '@prezzemolo/rap';
import Post from '../../models/post';
import ChannelWatching from '../../models/channel-watching';
import getFriends from '../../common/get-friends';
@@ -33,14 +34,15 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
return rej('cannot set since_id and max_id');
}
- // ID list of the user itself and other users who the user follows
- const followingIds = await getFriends(user._id);
-
- // Watchしているチャンネルを取得
- const watches = await ChannelWatching.find({
- user_id: user._id,
- // 削除されたドキュメントは除く
- deleted_at: { $exists: false }
+ const { followingIds, watchChannelIds } = await rap({
+ // ID list of the user itself and other users who the user follows
+ followingIds: getFriends(user._id),
+ // Watchしているチャンネルを取得
+ watchChannelIds: ChannelWatching.find({
+ user_id: user._id,
+ // 削除されたドキュメントは除く
+ deleted_at: { $exists: false }
+ }).then(watches => watches.map(w => w.channel_id))
});
//#region Construct query
@@ -65,7 +67,7 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
}, {
// Watchしているチャンネルへの投稿
channel_id: {
- $in: watches.map(w => w.channel_id)
+ $in: watchChannelIds
}
}]
} as any;
@@ -90,7 +92,5 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
});
// Serialize
- res(await Promise.all(timeline.map(async post =>
- await serialize(post, user)
- )));
+ res(Promise.all(timeline.map(post => serialize(post, user))));
});
diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts
index b4e2ab064a..4c750f4c6b 100644
--- a/src/api/serializers/drive-file.ts
+++ b/src/api/serializers/drive-file.ts
@@ -31,44 +31,37 @@ export default (
if (mongo.ObjectID.prototype.isPrototypeOf(file)) {
_file = await DriveFile.findOne({
_id: file
- }, {
- fields: {
- data: false
- }
- });
+ });
} else if (typeof file === 'string') {
_file = await DriveFile.findOne({
_id: new mongo.ObjectID(file)
- }, {
- fields: {
- data: false
- }
- });
+ });
} else {
_file = deepcopy(file);
}
- // Rename _id to id
- _file.id = _file._id;
- delete _file._id;
+ // rendered target
+ let _target: any = {};
- delete _file.data;
+ _target.id = _file._id;
- _file.url = `${config.drive_url}/${_file.id}/${encodeURIComponent(_file.name)}`;
+ _target = Object.assign(_target, _file.metadata);
- if (opts.detail && _file.folder_id) {
+ _target.url = `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`;
+
+ if (opts.detail && _target.folder_id) {
// Populate folder
- _file.folder = await serializeDriveFolder(_file.folder_id, {
+ _target.folder = await serializeDriveFolder(_target.folder_id, {
detail: true
});
}
- if (opts.detail && _file.tags) {
+ if (opts.detail && _target.tags) {
// Populate tags
- _file.tags = await _file.tags.map(async (tag: any) =>
+ _target.tags = await _target.tags.map(async (tag: any) =>
await serializeDriveTag(tag)
);
}
- resolve(_file);
+ resolve(_target);
});
diff --git a/src/api/serializers/drive-folder.ts b/src/api/serializers/drive-folder.ts
index a428464108..3b5f61aeed 100644
--- a/src/api/serializers/drive-folder.ts
+++ b/src/api/serializers/drive-folder.ts
@@ -44,7 +44,9 @@ const self = (
});
const childFilesCount = await DriveFile.count({
- folder_id: _folder.id
+ metadata: {
+ folder_id: _folder.id
+ }
});
_folder.folders_count = childFoldersCount;
From d0dab265f40a37cd715b7d4b64a364c78a7a35b9 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:27:16 +0900
Subject: [PATCH 11/29] serializers - drive-file: add created_at field by
uploadedDate
---
src/api/serializers/drive-file.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts
index 4c750f4c6b..f98cdaa599 100644
--- a/src/api/serializers/drive-file.ts
+++ b/src/api/serializers/drive-file.ts
@@ -44,6 +44,7 @@ export default (
let _target: any = {};
_target.id = _file._id;
+ _target.created_at = _file.uploadDate
_target = Object.assign(_target, _file.metadata);
From a5160a1bbaa3dd75d7ef45b305a90020317e95a8 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:35:20 +0900
Subject: [PATCH 12/29] fileserver - support DriveFile w/ GridFS
---
src/file/server.ts | 23 +++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/src/file/server.ts b/src/file/server.ts
index ee67cf7860..bd29e13c5c 100644
--- a/src/file/server.ts
+++ b/src/file/server.ts
@@ -9,7 +9,7 @@ import * as cors from 'cors';
import * as mongodb from 'mongodb';
import * as gm from 'gm';
-import File from '../api/models/drive-file';
+import DriveFile, { getGridFSBucket } from '../api/models/drive-file';
/**
* Init app
@@ -97,17 +97,28 @@ app.get('/:id', async (req, res) => {
return;
}
- const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) });
+ const fileId = new mongodb.ObjectID(req.params.id)
+ const file = await DriveFile.findOne({ _id: fileId });
if (file == null) {
res.status(404).sendFile(`${__dirname} / assets / dummy.png`);
return;
- } else if (file.data == null) {
- res.sendStatus(400);
- return;
}
- send(file.data.buffer, file.type, req, res);
+ const bucket = await getGridFSBucket()
+
+ const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
+ const chunks = []
+ const readableStream = bucket.openDownloadStream(id)
+ readableStream.on('data', chunk => {
+ chunks.push(chunk);
+ })
+ readableStream.on('end', () => {
+ resolve(Buffer.concat(chunks))
+ })
+ }))(fileId)
+
+ send(buffer, file.metadata.type, req, res);
});
app.get('/:id/:name', async (req, res) => {
From 2ce3179d5000501391b020dd98385aab9fed8094 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:37:04 +0900
Subject: [PATCH 13/29] fileserver - fix dummy path
---
src/file/server.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/file/server.ts b/src/file/server.ts
index bd29e13c5c..068e88546b 100644
--- a/src/file/server.ts
+++ b/src/file/server.ts
@@ -101,7 +101,7 @@ app.get('/:id', async (req, res) => {
const file = await DriveFile.findOne({ _id: fileId });
if (file == null) {
- res.status(404).sendFile(`${__dirname} / assets / dummy.png`);
+ res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
return;
}
From 28a39bccf96549a35ef77c10dce5f90f9f8cc654 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:39:16 +0900
Subject: [PATCH 14/29] file-server - support new DriveFile w/ GridFS on
'/:id/:name'
---
src/file/server.ts | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/src/file/server.ts b/src/file/server.ts
index 068e88546b..f38599b89c 100644
--- a/src/file/server.ts
+++ b/src/file/server.ts
@@ -128,17 +128,28 @@ app.get('/:id/:name', async (req, res) => {
return;
}
- const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) });
+ const fileId = new mongodb.ObjectID(req.params.id)
+ const file = await DriveFile.findOne({ _id: fileId });
if (file == null) {
res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
return;
- } else if (file.data == null) {
- res.sendStatus(400);
- return;
}
- send(file.data.buffer, file.type, req, res);
+ const bucket = await getGridFSBucket()
+
+ const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
+ const chunks = []
+ const readableStream = bucket.openDownloadStream(id)
+ readableStream.on('data', chunk => {
+ chunks.push(chunk);
+ })
+ readableStream.on('end', () => {
+ resolve(Buffer.concat(chunks))
+ })
+ }))(fileId)
+
+ send(buffer, file.metadata.type, req, res);
});
module.exports = app;
From 0ee6d6592113c5b2df071f4451cb1c1697b59d61 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:45:21 +0900
Subject: [PATCH 15/29] fix timeline
---
src/api/endpoints/posts/timeline.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts
index 496de62b69..19578e59b1 100644
--- a/src/api/endpoints/posts/timeline.ts
+++ b/src/api/endpoints/posts/timeline.ts
@@ -92,5 +92,5 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
});
// Serialize
- res(Promise.all(timeline.map(post => serialize(post, user))));
+ res(await Promise.all(timeline.map(post => serialize(post, user))));
});
From 7553c6dd38c6f8574894a009238d946d50c53477 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:52:09 +0900
Subject: [PATCH 16/29] serializers - posts: no need Promise wrapping
---
src/api/serializers/post.ts | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index e1ab784359..d1dcb66002 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -22,13 +22,13 @@ import rap from '@prezzemolo/rap';
* @param options? serialize options
* @return response
*/
-const self = (
+const self = async (
post: string | mongo.ObjectID | IPost,
me?: string | mongo.ObjectID | IUser,
options?: {
detail: boolean
}
-) => new Promise<any>(async (resolve, reject) => {
+) => {
const opts = options || {
detail: true,
};
@@ -184,7 +184,7 @@ const self = (
// resolve promises in _post object
_post = await rap(_post);
- resolve(_post);
-});
+ return _post;
+};
export default self;
From 7b1fc2c5d62e229542e9411a29e078236a9d96db Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:55:47 +0900
Subject: [PATCH 17/29] api - endpoint:timeline: unneed promise wrapping
---
src/api/endpoints/posts/timeline.ts | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts
index 19578e59b1..978825a109 100644
--- a/src/api/endpoints/posts/timeline.ts
+++ b/src/api/endpoints/posts/timeline.ts
@@ -16,22 +16,22 @@ import serialize from '../../serializers/post';
* @param {any} app
* @return {Promise<any>}
*/
-module.exports = (params, user, app) => new Promise(async (res, rej) => {
+module.exports = async (params, user, app) => {
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
- if (limitErr) return rej('invalid limit param');
+ if (limitErr) throw 'invalid limit param';
// Get 'since_id' parameter
const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
- if (sinceIdErr) return rej('invalid since_id param');
+ if (sinceIdErr) throw 'invalid since_id param';
// Get 'max_id' parameter
const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
- if (maxIdErr) return rej('invalid max_id param');
+ if (maxIdErr) throw 'invalid max_id param';
// Check if both of since_id and max_id is specified
if (sinceId && maxId) {
- return rej('cannot set since_id and max_id');
+ throw 'cannot set since_id and max_id';
}
const { followingIds, watchChannelIds } = await rap({
@@ -92,5 +92,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
});
// Serialize
- res(await Promise.all(timeline.map(post => serialize(post, user))));
-});
+ const _timeline = await Promise.all(timeline.map(post => serialize(post, user)))
+ return _timeline
+};
From b50813649afed671b75189551342b179d8cd60f7 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 15:58:39 +0900
Subject: [PATCH 18/29] serializers - posts: fix awaiting
---
src/api/serializers/post.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index d1dcb66002..5788b226f4 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -94,7 +94,7 @@ const self = async (
if (opts.detail) {
// Get previous post info
_post.prev = (async () => {
- const prev = Post.findOne({
+ const prev = await Post.findOne({
user_id: _post.user_id,
_id: {
$lt: id
From 5279d062df205514f1f3cf95e3aab4fee425a3e4 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:09:51 +0900
Subject: [PATCH 19/29] fix
---
src/api/endpoints/drive/files.ts | 18 +++++++++---------
src/api/endpoints/drive/files/show.ts | 14 ++++++++------
src/api/endpoints/drive/folders/find.ts | 3 +--
src/api/serializers/drive-file.ts | 2 ++
4 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts
index eb0bfe6ba5..41687c4993 100644
--- a/src/api/endpoints/drive/files.ts
+++ b/src/api/endpoints/drive/files.ts
@@ -13,27 +13,27 @@ import serialize from '../../serializers/drive-file';
* @param {any} app
* @return {Promise<any>}
*/
-module.exports = (params, user, app) => new Promise(async (res, rej) => {
+module.exports = async (params, user, app) => {
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
- if (limitErr) return rej('invalid limit param');
+ if (limitErr) throw 'invalid limit param';
// Get 'since_id' parameter
const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
- if (sinceIdErr) return rej('invalid since_id param');
+ if (sinceIdErr) throw 'invalid since_id param';
// Get 'max_id' parameter
const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
- if (maxIdErr) return rej('invalid max_id param');
+ if (maxIdErr) throw 'invalid max_id param';
// Check if both of since_id and max_id is specified
if (sinceId && maxId) {
- return rej('cannot set since_id and max_id');
+ throw 'cannot set since_id and max_id';
}
// Get 'folder_id' parameter
const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
- if (folderIdErr) return rej('invalid folder_id param');
+ if (folderIdErr) throw 'invalid folder_id param';
// Construct query
const sort = {
@@ -64,6 +64,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
});
// Serialize
- res(await Promise.all(files.map(async file =>
- await serialize(file))));
-});
+ const _files = await Promise.all(files.map(file => serialize(file)));
+ return _files
+};
diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts
index 9135a04c57..8830346008 100644
--- a/src/api/endpoints/drive/files/show.ts
+++ b/src/api/endpoints/drive/files/show.ts
@@ -12,10 +12,10 @@ import serialize from '../../../serializers/drive-file';
* @param {any} user
* @return {Promise<any>}
*/
-module.exports = (params, user) => new Promise(async (res, rej) => {
+module.exports = async (params, user) => {
// Get 'file_id' parameter
const [fileId, fileIdErr] = $(params.file_id).id().$;
- if (fileIdErr) return rej('invalid file_id param');
+ if (fileIdErr) throw 'invalid file_id param';
// Fetch file
const file = await DriveFile
@@ -27,11 +27,13 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
});
if (file === null) {
- return rej('file-not-found');
+ throw 'file-not-found';
}
// Serialize
- res(await serialize(file, {
+ const _file = await serialize(file, {
detail: true
- }));
-});
+ });
+
+ return _file
+};
diff --git a/src/api/endpoints/drive/folders/find.ts b/src/api/endpoints/drive/folders/find.ts
index cdf055839a..a5eb8e015d 100644
--- a/src/api/endpoints/drive/folders/find.ts
+++ b/src/api/endpoints/drive/folders/find.ts
@@ -30,6 +30,5 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
});
// Serialize
- res(await Promise.all(folders.map(async folder =>
- await serialize(folder))));
+ res(await Promise.all(folders.map(folder => serialize(folder))));
});
diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts
index f98cdaa599..9858c3b3c7 100644
--- a/src/api/serializers/drive-file.ts
+++ b/src/api/serializers/drive-file.ts
@@ -25,6 +25,8 @@ export default (
detail: false
}, options);
+ if (!file) return reject('invalid file arg.')
+
let _file: any;
// Populate the file if 'file' is ID
From b266ed3e4f98ab16d95e52cff517d6519b78742a Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:11:24 +0900
Subject: [PATCH 20/29] fix
---
src/api/serializers/drive-file.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts
index 9858c3b3c7..e749f80387 100644
--- a/src/api/serializers/drive-file.ts
+++ b/src/api/serializers/drive-file.ts
@@ -25,8 +25,6 @@ export default (
detail: false
}, options);
- if (!file) return reject('invalid file arg.')
-
let _file: any;
// Populate the file if 'file' is ID
@@ -42,6 +40,8 @@ export default (
_file = deepcopy(file);
}
+ if (!_file) return reject('invalid file arg.')
+
// rendered target
let _target: any = {};
From 64be0d6deddef4b8caced377dc22f94425cc4358 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:22:18 +0900
Subject: [PATCH 21/29] =?UTF-8?q?MongoDB=E3=81=AE=E9=9A=8E=E5=B1=A4?=
=?UTF-8?q?=E6=A7=8B=E9=80=A0=E6=A4=9C=E7=B4=A2=E3=81=AB=E9=96=A2=E3=81=99?=
=?UTF-8?q?=E3=82=8B=E6=80=9D=E3=81=84=E9=81=95=E3=81=84=E3=81=AE=E4=BF=AE?=
=?UTF-8?q?=E6=AD=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
src/api/endpoints/drive.ts | 2 +-
src/api/endpoints/drive/files.ts | 6 ++----
src/api/endpoints/drive/files/find.ts | 8 +++-----
src/api/endpoints/drive/files/show.ts | 4 +---
src/api/endpoints/drive/files/update.ts | 19 +++++--------------
.../endpoints/messaging/messages/create.ts | 4 +---
src/api/endpoints/posts/create.ts | 4 +---
src/api/serializers/drive-folder.ts | 4 +---
8 files changed, 15 insertions(+), 36 deletions(-)
diff --git a/src/api/endpoints/drive.ts b/src/api/endpoints/drive.ts
index b9c4e3e506..d92473633a 100644
--- a/src/api/endpoints/drive.ts
+++ b/src/api/endpoints/drive.ts
@@ -14,7 +14,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Calculate drive usage
const usage = ((await DriveFile
.aggregate([
- { $match: { metadata: { user_id: user._id } } },
+ { $match: { 'metadata.user_id': user._id } },
{
$project: {
length: true
diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts
index 41687c4993..035916b309 100644
--- a/src/api/endpoints/drive/files.ts
+++ b/src/api/endpoints/drive/files.ts
@@ -40,10 +40,8 @@ module.exports = async (params, user, app) => {
_id: -1
};
const query = {
- metadata: {
- user_id: user._id,
- folder_id: folderId
- }
+ 'metadata.user_id': user._id,
+ 'metadata.folder_id': folderId
} as any;
if (sinceId) {
sort._id = 1;
diff --git a/src/api/endpoints/drive/files/find.ts b/src/api/endpoints/drive/files/find.ts
index 255faf94ec..1c818131d7 100644
--- a/src/api/endpoints/drive/files/find.ts
+++ b/src/api/endpoints/drive/files/find.ts
@@ -24,11 +24,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Issue query
const files = await DriveFile
.find({
- metadata: {
- name: name,
- user_id: user._id,
- folder_id: folderId
- }
+ 'metadata.name': name,
+ 'metadata.user_id': user._id,
+ 'metadata.folder_id': folderId
});
// Serialize
diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts
index 8830346008..0a19b19939 100644
--- a/src/api/endpoints/drive/files/show.ts
+++ b/src/api/endpoints/drive/files/show.ts
@@ -21,9 +21,7 @@ module.exports = async (params, user) => {
const file = await DriveFile
.findOne({
_id: fileId,
- metadata: {
- user_id: user._id
- }
+ 'metadata.user_id': user._id
});
if (file === null) {
diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts
index c4d2673688..7a6d2562fb 100644
--- a/src/api/endpoints/drive/files/update.ts
+++ b/src/api/endpoints/drive/files/update.ts
@@ -20,19 +20,14 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
const [fileId, fileIdErr] = $(params.file_id).id().$;
if (fileIdErr) return rej('invalid file_id param');
- console.dir(user)
// Fetch file
const file = await DriveFile
.findOne({
_id: fileId,
- metadata: {
- user_id: user._id
- }
+ 'metadata.user_id': user._id
});
- console.dir(file)
-
if (file === null) {
return rej('file-not-found');
}
@@ -42,7 +37,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Get 'name' parameter
const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
if (nameErr) return rej('invalid name param');
- if (name) updateQuery.name = name;
+ if (name) updateQuery['metadata.name'] = name;
// Get 'folder_id' parameter
const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
@@ -50,7 +45,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (folderId !== undefined) {
if (folderId === null) {
- updateQuery.folder_id = null;
+ updateQuery['metadata.folder_id'] = null;
} else {
// Fetch folder
const folder = await DriveFolder
@@ -63,21 +58,17 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('folder-not-found');
}
- updateQuery.folder_id = folder._id;
+ updateQuery['metadata.folder_id'] = folder._id;
}
}
const updated = await DriveFile.update(file._id, {
- $set: { metadata: updateQuery }
+ $set: { updateQuery }
});
- console.dir(updated)
-
// Serialize
const fileObj = await serialize(updated);
- console.dir(fileObj)
-
// Response
res(fileObj);
diff --git a/src/api/endpoints/messaging/messages/create.ts b/src/api/endpoints/messaging/messages/create.ts
index 1d186268fb..149852c093 100644
--- a/src/api/endpoints/messaging/messages/create.ts
+++ b/src/api/endpoints/messaging/messages/create.ts
@@ -54,9 +54,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (fileId !== undefined) {
file = await DriveFile.findOne({
_id: fileId,
- metadata: {
- user_id: user._id
- }
+ 'metadata.user_id': user._id
});
if (file === null) {
diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts
index 1507639776..4f4b7e2e83 100644
--- a/src/api/endpoints/posts/create.ts
+++ b/src/api/endpoints/posts/create.ts
@@ -44,9 +44,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
// SELECT _id
const entity = await DriveFile.findOne({
_id: mediaId,
- metadata: {
- user_id: user._id
- }
+ 'metadata.user_id': user._id
});
if (entity === null) {
diff --git a/src/api/serializers/drive-folder.ts b/src/api/serializers/drive-folder.ts
index 3b5f61aeed..6ebf454a28 100644
--- a/src/api/serializers/drive-folder.ts
+++ b/src/api/serializers/drive-folder.ts
@@ -44,9 +44,7 @@ const self = (
});
const childFilesCount = await DriveFile.count({
- metadata: {
- folder_id: _folder.id
- }
+ 'metadata.folder_id': _folder.id
});
_folder.folders_count = childFoldersCount;
From 4c5a4d259738ba617bf29d2158d180cc5fa8401c Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:26:17 +0900
Subject: [PATCH 22/29] core - fix metadata searching
---
src/api/common/add-file-to-drive.ts | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index 376c470e93..1f882389ac 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -81,9 +81,7 @@ export default (
// Check if there is a file with the same hash
const much = await DriveFile.findOne({
md5: hash,
- metadata: {
- user_id: user._id
- }
+ 'metadata.user_id': user._id
});
if (much !== null) {
@@ -97,7 +95,7 @@ export default (
// Calculate drive usage
const usage = ((await DriveFile
.aggregate([
- { $match: { metadata: { user_id: user._id } } },
+ { $match: { 'metadata.user_id': user._id } },
{ $project: {
length: true
}},
From 04648db1c235b0de14d3e0a2dc83f9346d0408f8 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:29:13 +0900
Subject: [PATCH 23/29] remove console
---
src/api/common/add-file-to-drive.ts | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index 1f882389ac..dff2d52356 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -152,7 +152,6 @@ export default (
comment: comment,
properties: properties
});
- console.dir(file)
log(`drive file has been created ${file._id}`);
From d5cc4cc9c28eb6a981ce37859def97cd7c57abc6 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:32:01 +0900
Subject: [PATCH 24/29] fix lint (automattic)
---
src/api/common/add-file-to-drive.ts | 18 ++++++-------
src/api/endpoints/drive/files.ts | 2 +-
src/api/endpoints/drive/files/show.ts | 2 +-
src/api/endpoints/drive/files/update.ts | 3 +--
src/api/endpoints/posts/timeline.ts | 4 +--
src/api/models/drive-file.ts | 10 +++----
src/api/serializers/drive-file.ts | 4 +--
src/db/mongodb.ts | 18 ++++++-------
src/file/server.ts | 36 ++++++++++++-------------
9 files changed, 48 insertions(+), 49 deletions(-)
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index dff2d52356..f9c22ccacd 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -14,16 +14,16 @@ import { Duplex } from 'stream';
const log = debug('misskey:register-drive-file');
const addToGridFS = (name, binary, metadata): Promise<any> => new Promise(async (resolve, reject) => {
- const dataStream = new Duplex()
- dataStream.push(binary)
- dataStream.push(null)
+ const dataStream = new Duplex();
+ dataStream.push(binary);
+ dataStream.push(null);
- const bucket = await getGridFSBucket()
- const writeStream = bucket.openUploadStream(name, { metadata })
- writeStream.once('finish', (doc) => { resolve(doc) })
- writeStream.on('error', reject)
- dataStream.pipe(writeStream)
-})
+ const bucket = await getGridFSBucket();
+ const writeStream = bucket.openUploadStream(name, { metadata });
+ writeStream.once('finish', (doc) => { resolve(doc); });
+ writeStream.on('error', reject);
+ dataStream.pipe(writeStream);
+});
/**
* Add file to drive
diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts
index 035916b309..53b48a8bec 100644
--- a/src/api/endpoints/drive/files.ts
+++ b/src/api/endpoints/drive/files.ts
@@ -63,5 +63,5 @@ module.exports = async (params, user, app) => {
// Serialize
const _files = await Promise.all(files.map(file => serialize(file)));
- return _files
+ return _files;
};
diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts
index 0a19b19939..3c7cf774f9 100644
--- a/src/api/endpoints/drive/files/show.ts
+++ b/src/api/endpoints/drive/files/show.ts
@@ -33,5 +33,5 @@ module.exports = async (params, user) => {
detail: true
});
- return _file
+ return _file;
};
diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts
index 7a6d2562fb..4e56b30ace 100644
--- a/src/api/endpoints/drive/files/update.ts
+++ b/src/api/endpoints/drive/files/update.ts
@@ -20,7 +20,6 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
const [fileId, fileIdErr] = $(params.file_id).id().$;
if (fileIdErr) return rej('invalid file_id param');
-
// Fetch file
const file = await DriveFile
.findOne({
@@ -32,7 +31,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('file-not-found');
}
- const updateQuery: any = {}
+ const updateQuery: any = {};
// Get 'name' parameter
const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts
index 978825a109..203413e23a 100644
--- a/src/api/endpoints/posts/timeline.ts
+++ b/src/api/endpoints/posts/timeline.ts
@@ -92,6 +92,6 @@ module.exports = async (params, user, app) => {
});
// Serialize
- const _timeline = await Promise.all(timeline.map(post => serialize(post, user)))
- return _timeline
+ const _timeline = await Promise.all(timeline.map(post => serialize(post, user)));
+ return _timeline;
};
diff --git a/src/api/models/drive-file.ts b/src/api/models/drive-file.ts
index 79a87f6572..8968d065cd 100644
--- a/src/api/models/drive-file.ts
+++ b/src/api/models/drive-file.ts
@@ -8,14 +8,14 @@ const collection = monkDb.get('drive_files.files');
export default collection as any; // fuck type definition
const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
- const db = await nativeDbConn()
+ const db = await nativeDbConn();
const bucket = new mongodb.GridFSBucket(db, {
bucketName: 'drive_files'
- })
- return bucket
-}
+ });
+ return bucket;
+};
-export { getGridFSBucket }
+export { getGridFSBucket };
export function validateFileName(name: string): boolean {
return (
diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts
index e749f80387..2af7db5726 100644
--- a/src/api/serializers/drive-file.ts
+++ b/src/api/serializers/drive-file.ts
@@ -40,13 +40,13 @@ export default (
_file = deepcopy(file);
}
- if (!_file) return reject('invalid file arg.')
+ if (!_file) return reject('invalid file arg.');
// rendered target
let _target: any = {};
_target.id = _file._id;
- _target.created_at = _file.uploadDate
+ _target.created_at = _file.uploadDate;
_target = Object.assign(_target, _file.metadata);
diff --git a/src/db/mongodb.ts b/src/db/mongodb.ts
index 75f1a1d3c6..c978e6460f 100644
--- a/src/db/mongodb.ts
+++ b/src/db/mongodb.ts
@@ -16,7 +16,7 @@ export default db;
/**
* MongoDB native module (officialy)
*/
-import * as mongodb from 'mongodb'
+import * as mongodb from 'mongodb';
let mdb: mongodb.Db;
@@ -25,14 +25,14 @@ const nativeDbConn = async (): Promise<mongodb.Db> => {
const db = await ((): Promise<mongodb.Db> => new Promise((resolve, reject) => {
mongodb.MongoClient.connect(uri, (e, db) => {
- if (e) return reject(e)
- resolve(db)
- })
- }))()
+ if (e) return reject(e);
+ resolve(db);
+ });
+ }))();
- mdb = db
+ mdb = db;
- return db
-}
+ return db;
+};
-export { nativeDbConn }
+export { nativeDbConn };
diff --git a/src/file/server.ts b/src/file/server.ts
index f38599b89c..375f29487d 100644
--- a/src/file/server.ts
+++ b/src/file/server.ts
@@ -97,7 +97,7 @@ app.get('/:id', async (req, res) => {
return;
}
- const fileId = new mongodb.ObjectID(req.params.id)
+ const fileId = new mongodb.ObjectID(req.params.id);
const file = await DriveFile.findOne({ _id: fileId });
if (file == null) {
@@ -105,18 +105,18 @@ app.get('/:id', async (req, res) => {
return;
}
- const bucket = await getGridFSBucket()
+ const bucket = await getGridFSBucket();
const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
- const chunks = []
- const readableStream = bucket.openDownloadStream(id)
- readableStream.on('data', chunk => {
+ const chunks = [];
+ const readableStream = bucket.openDownloadStream(id);
+ readableStream.on('data', chunk => {
chunks.push(chunk);
- })
+ });
readableStream.on('end', () => {
- resolve(Buffer.concat(chunks))
- })
- }))(fileId)
+ resolve(Buffer.concat(chunks));
+ });
+ }))(fileId);
send(buffer, file.metadata.type, req, res);
});
@@ -128,7 +128,7 @@ app.get('/:id/:name', async (req, res) => {
return;
}
- const fileId = new mongodb.ObjectID(req.params.id)
+ const fileId = new mongodb.ObjectID(req.params.id);
const file = await DriveFile.findOne({ _id: fileId });
if (file == null) {
@@ -136,18 +136,18 @@ app.get('/:id/:name', async (req, res) => {
return;
}
- const bucket = await getGridFSBucket()
+ const bucket = await getGridFSBucket();
const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
- const chunks = []
- const readableStream = bucket.openDownloadStream(id)
- readableStream.on('data', chunk => {
+ const chunks = [];
+ const readableStream = bucket.openDownloadStream(id);
+ readableStream.on('data', chunk => {
chunks.push(chunk);
- })
+ });
readableStream.on('end', () => {
- resolve(Buffer.concat(chunks))
- })
- }))(fileId)
+ resolve(Buffer.concat(chunks));
+ });
+ }))(fileId);
send(buffer, file.metadata.type, req, res);
});
From 3be69a8cb7bacca181fa400f234fd77c1d1d5bde Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:49:07 +0900
Subject: [PATCH 25/29] /drive/files/update - return collectly value
---
src/api/endpoints/drive/files/update.ts | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts
index 4e56b30ace..d7b858c2ba 100644
--- a/src/api/endpoints/drive/files/update.ts
+++ b/src/api/endpoints/drive/files/update.ts
@@ -31,12 +31,10 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('file-not-found');
}
- const updateQuery: any = {};
-
// Get 'name' parameter
const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
if (nameErr) return rej('invalid name param');
- if (name) updateQuery['metadata.name'] = name;
+ if (name) file.metadata.name = name;
// Get 'folder_id' parameter
const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
@@ -44,7 +42,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (folderId !== undefined) {
if (folderId === null) {
- updateQuery['metadata.folder_id'] = null;
+ file.metadata.folder_id = null;
} else {
// Fetch folder
const folder = await DriveFolder
@@ -57,16 +55,19 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('folder-not-found');
}
- updateQuery['metadata.folder_id'] = folder._id;
+ file.metadata.folder_id = folder._id;
}
}
- const updated = await DriveFile.update(file._id, {
- $set: { updateQuery }
+ await DriveFile.update(file._id, {
+ $set: {
+ 'metadata.name': file.metadata.name,
+ 'metadata.folder_id': file.metadata.folder_id
+ }
});
// Serialize
- const fileObj = await serialize(updated);
+ const fileObj = await serialize(file);
// Response
res(fileObj);
From 73bb81de8f17fe603dfde57ae70aa61669161bfc Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 16:59:09 +0900
Subject: [PATCH 26/29] update test for GridFS
---
test/api.js | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/test/api.js b/test/api.js
index b43eb7ff62..c0da9d6c5b 100644
--- a/test/api.js
+++ b/test/api.js
@@ -1152,9 +1152,12 @@ async function insertHimawari(opts) {
}
async function insertDriveFile(opts) {
- return await db.get('drive_files').insert(Object.assign({
- name: 'strawberry-pasta.png'
- }, opts));
+ return await db.get('drive_files.files').insert({
+ length: opts.datasize,
+ metadata: Object.assign({
+ name: 'strawberry-pasta.png'
+ }, opts)
+ });
}
async function insertDriveFolder(opts) {
From 26602dcd209198dead66081f54b1800627e0bff8 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 17:57:03 +0900
Subject: [PATCH 27/29] migration - add GridFS migration
---
tools/migration/use-gridfs.js | 49 +++++++++++++++++++++++++++++++++++
1 file changed, 49 insertions(+)
create mode 100644 tools/migration/use-gridfs.js
diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js
new file mode 100644
index 0000000000..d41514416c
--- /dev/null
+++ b/tools/migration/use-gridfs.js
@@ -0,0 +1,49 @@
+// for Node.js interpret
+
+const { default: db } = require('../../built/db/mongodb')
+const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file')
+const { Duplex } = require('stream')
+
+const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => {
+ const writeStream = bucket.openUploadStreamWithId(...rest)
+
+ const dataStream = new Duplex()
+ dataStream.push(buffer)
+ dataStream.push(null)
+
+ writeStream.once('finish', resolve)
+ writeStream.on('error', reject)
+
+ dataStream.pipe(writeStream)
+})
+
+const migrateToGridFS = async (doc) => {
+ const id = doc._id
+ const buffer = doc.data.buffer
+ const created_at = doc.created_at
+
+ delete doc._id
+ delete doc.created_at
+ delete doc.datasize
+ delete doc.hash
+ delete doc.data
+
+ const bucket = await getGridFSBucket()
+ const added = await writeToGridFS(bucket, buffer, id, `${id}/${doc.name}`, { metadata: doc })
+
+ const result = await DriveFile.update(id, {
+ $set: {
+ uploadDate: created_at
+ }
+ })
+
+ return added && result.ok === 1
+}
+
+const main = async () => {
+ const docs = await db.get('drive_files').find()
+ const all = await Promise.all(docs.map(migrateToGridFS))
+ return all
+}
+
+main().then(console.dir).catch(console.error)
From c1fc3b9f6ec176999932958a7856d160317b7762 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 18:30:49 +0900
Subject: [PATCH 28/29] add safety guard to serializers & fix importing
uncorrect serializer
---
src/api/endpoints/drive/folders/update.ts | 2 +-
src/api/serializers/post.ts | 2 ++
src/api/serializers/user.ts | 2 ++
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/api/endpoints/drive/folders/update.ts b/src/api/endpoints/drive/folders/update.ts
index eec2757878..4f2e3d2a7a 100644
--- a/src/api/endpoints/drive/folders/update.ts
+++ b/src/api/endpoints/drive/folders/update.ts
@@ -4,7 +4,7 @@
import $ from 'cafy';
import DriveFolder from '../../../models/drive-folder';
import { isValidFolderName } from '../../../models/drive-folder';
-import serialize from '../../../serializers/drive-file';
+import serialize from '../../../serializers/drive-folder';
import event from '../../../event';
/**
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index 5788b226f4..5a63384f0e 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -57,6 +57,8 @@ const self = async (
_post = deepcopy(post);
}
+ if (!_post) throw 'invalid post arg.';
+
const id = _post._id;
// Rename _id to id
diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts
index d00f073897..0d24d6cc04 100644
--- a/src/api/serializers/user.ts
+++ b/src/api/serializers/user.ts
@@ -56,6 +56,8 @@ export default (
_user = deepcopy(user);
}
+ if (!_user) return reject('invalid user arg.');
+
// Me
const meId: mongo.ObjectID = me
? mongo.ObjectID.prototype.isPrototypeOf(me)
From d7e1ffb0055f0786a707015350a14351b8a0fbf0 Mon Sep 17 00:00:00 2001
From: otofune <otofune@gmail.com>
Date: Mon, 6 Nov 2017 18:38:59 +0900
Subject: [PATCH 29/29] remove whitespace
---
src/api/serializers/post.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts
index 5a63384f0e..03fd120772 100644
--- a/src/api/serializers/post.ts
+++ b/src/api/serializers/post.ts
@@ -57,7 +57,7 @@ const self = async (
_post = deepcopy(post);
}
- if (!_post) throw 'invalid post arg.';
+ if (!_post) throw 'invalid post arg.';
const id = _post._id;