diff --git a/src/misc/get-drive-file-url.ts b/src/misc/get-drive-file-url.ts index 0fe467261eda83568c52f32afdee577a7dc76564..6ab7bfdb1be175c24bdd3cd0693b69394844fae6 100644 --- a/src/misc/get-drive-file-url.ts +++ b/src/misc/get-drive-file-url.ts @@ -6,15 +6,24 @@ export default function(file: IDriveFile, thumbnail = false): string { if (file.metadata.withoutChunks) { if (thumbnail) { - return file.metadata.thumbnailUrl || file.metadata.url; + return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || file.metadata.url; } else { - return file.metadata.url; + return file.metadata.webpublicUrl || file.metadata.url; } } else { if (thumbnail) { return `${config.drive_url}/${file._id}?thumbnail`; } else { - return `${config.drive_url}/${file._id}`; + return `${config.drive_url}/${file._id}?web`; } } } + +export function getOriginalUrl(file: IDriveFile) { + if (file.metadata && file.metadata.url) { + return file.metadata.url; + } + + const accessKey = file.metadata ? file.metadata.accessKey : null; + return `${config.drive_url}/${file._id}${accessKey ? '?original=' + accessKey : ''}`; +} diff --git a/src/models/drive-file-webpublic.ts b/src/models/drive-file-webpublic.ts new file mode 100644 index 0000000000000000000000000000000000000000..d087c355d39a8bb38ceb5d37a4d375a502d26bb4 --- /dev/null +++ b/src/models/drive-file-webpublic.ts @@ -0,0 +1,29 @@ +import * as mongo from 'mongodb'; +import monkDb, { nativeDbConn } from '../db/mongodb'; + +const DriveFileWebpublic = monkDb.get<IDriveFileWebpublic>('driveFileWebpublics.files'); +DriveFileWebpublic.createIndex('metadata.originalId', { sparse: true, unique: true }); +export default DriveFileWebpublic; + +export const DriveFileWebpublicChunk = monkDb.get('driveFileWebpublics.chunks'); + +export const getDriveFileWebpublicBucket = async (): Promise<mongo.GridFSBucket> => { + const db = await nativeDbConn(); + const bucket = new mongo.GridFSBucket(db, { + bucketName: 'driveFileWebpublics' + }); + return bucket; +}; + +export type IMetadata = { + originalId: mongo.ObjectID; +}; + +export type IDriveFileWebpublic = { + _id: mongo.ObjectID; + uploadDate: Date; + md5: string; + filename: string; + contentType: string; + metadata: IMetadata; +}; diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index d0c0905fc207a45c3cbf596dfc523dcbd24667b1..e4c15980498c237e8297a21484c09c622101f14e 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -3,7 +3,7 @@ const deepcopy = require('deepcopy'); import { pack as packFolder } from './drive-folder'; import monkDb, { nativeDbConn } from '../db/mongodb'; import isObjectId from '../misc/is-objectid'; -import getDriveFileUrl from '../misc/get-drive-file-url'; +import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url'; const DriveFile = monkDb.get<IDriveFile>('driveFiles.files'); DriveFile.createIndex('md5'); @@ -28,21 +28,48 @@ export type IMetadata = { _user: any; folderId: mongo.ObjectID; comment: string; + + /** + * リモートインスタンスã‹ã‚‰å–å¾—ã—ãŸå ´åˆã®å…ƒURL + */ uri?: string; + + /** + * URL for web(生æˆã•ã‚Œã¦ã„ã‚‹å ´åˆ) or original + * * オブジェクトストレージを利用ã—ã¦ã„ã‚‹ or リモートサーãƒãƒ¼ã¸ã®ç›´ãƒªãƒ³ã‚¯ã§ã‚ã‚‹ å ´åˆã®ã¿ + */ url?: string; + + /** + * URL for thumbnail (thumbnailãŒãªã‘ã‚Œã°ãªã—) + * * オブジェクトストレージを利用ã—ã¦ã„ã‚‹ or リモートサーãƒãƒ¼ã¸ã®ç›´ãƒªãƒ³ã‚¯ã§ã‚ã‚‹ å ´åˆã®ã¿ + */ thumbnailUrl?: string; + + /** + * URL for original (web用ãŒç”Ÿæˆã•ã‚Œã¦ãªã„å ´åˆã¯urlãŒoriginalを指ã™) + * * オブジェクトストレージを利用ã—ã¦ã„ã‚‹ or リモートサーãƒãƒ¼ã¸ã®ç›´ãƒªãƒ³ã‚¯ã§ã‚ã‚‹ å ´åˆã®ã¿ + */ + webpublicUrl?: string; + + accessKey?: string; + src?: string; deletedAt?: Date; /** - * ã“ã®ãƒ•ã‚¡ã‚¤ãƒ«ã®ä¸èº«ãƒ‡ãƒ¼ã‚¿ãŒMongoDB内ã«ä¿å˜ã•ã‚Œã¦ã„ã‚‹ã®ã‹å¦ã‹ + * ã“ã®ãƒ•ã‚¡ã‚¤ãƒ«ã®ä¸èº«ãƒ‡ãƒ¼ã‚¿ãŒMongoDB内ã«ä¿å˜ã•ã‚Œã¦ã„ãªã„ã‹å¦ã‹ * オブジェクトストレージを利用ã—ã¦ã„ã‚‹ or リモートサーãƒãƒ¼ã¸ã®ç›´ãƒªãƒ³ã‚¯ã§ã‚ã‚‹ - * ãªå ´åˆã¯ false ã«ãªã‚Šã¾ã™ + * ãªå ´åˆã¯ true ã«ãªã‚Šã¾ã™ */ withoutChunks?: boolean; storage?: string; - storageProps?: any; + + /*** + * ObjectStorage ã®æ ¼ç´å…ˆã®æƒ…å ± + */ + storageProps?: IStorageProps; isSensitive?: boolean; /** @@ -56,6 +83,25 @@ export type IMetadata = { isRemote?: boolean; }; +export type IStorageProps = { + /** + * ObjectStorage key for original + */ + key: string; + + /*** + * ObjectStorage key for thumbnail (thumbnailãŒãªã‘ã‚Œã°ãªã—) + */ + thumbnailKey?: string; + + /*** + * ObjectStorage key for webpublic (webpublicãŒãªã‘ã‚Œã°ãªã—) + */ + webpublicKey?: string; + + id?: string; +}; + export type IDriveFile = { _id: mongo.ObjectID; uploadDate: Date; @@ -83,7 +129,8 @@ export function validateFileName(name: string): boolean { export const packMany = ( files: any[], options?: { - detail: boolean + detail?: boolean + self?: boolean, } ) => { return Promise.all(files.map(f => pack(f, options))); @@ -95,11 +142,13 @@ export const packMany = ( export const pack = ( file: any, options?: { - detail: boolean + detail?: boolean, + self?: boolean, } ) => new Promise<any>(async (resolve, reject) => { const opts = Object.assign({ - detail: false + detail: false, + self: false }, options); let _file: any; @@ -165,5 +214,9 @@ export const pack = ( delete _target.isRemote; delete _target._user; + if (opts.self) { + _target.url = getOriginalUrl(_file); + } + resolve(_target); }); diff --git a/src/server/api/endpoints/drive/files.ts b/src/server/api/endpoints/drive/files.ts index 27f101562dd0a64e8c66556381607be23b49b8fd..20955e0e4e4c85b1de808afb08d4a46b7de2d2c5 100644 --- a/src/server/api/endpoints/drive/files.ts +++ b/src/server/api/endpoints/drive/files.ts @@ -77,5 +77,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { sort: sort }); - res(await packMany(files)); + res(await packMany(files, { detail: false, self: true })); })); diff --git a/src/server/api/endpoints/drive/files/check_existence.ts b/src/server/api/endpoints/drive/files/check_existence.ts index d3ba4b386d0c3d397621a7c3026fad528b892d38..6e986d4170c34a373b0662f5f7cc060354633603 100644 --- a/src/server/api/endpoints/drive/files/check_existence.ts +++ b/src/server/api/endpoints/drive/files/check_existence.ts @@ -32,6 +32,6 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { if (file === null) { res({ file: null }); } else { - res({ file: await pack(file) }); + res({ file: await pack(file, { self: true }) }); } })); diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts index 53c62dd86815595e6a4e1fb477283188c9e2d628..0660627f082c682bca14d1b0997ce772b788c178 100644 --- a/src/server/api/endpoints/drive/files/create.ts +++ b/src/server/api/endpoints/drive/files/create.ts @@ -74,7 +74,7 @@ export default define(meta, (ps, user, app, file, cleanup) => new Promise(async cleanup(); - res(pack(driveFile)); + res(pack(driveFile, { self: true })); } catch (e) { console.error(e); diff --git a/src/server/api/endpoints/drive/files/find.ts b/src/server/api/endpoints/drive/files/find.ts index 8bc392fefe8cf9d3303047c3d7810bc768bb8055..25135e83a29b6eabae0fdcd88ad33f4f9acb89e5 100644 --- a/src/server/api/endpoints/drive/files/find.ts +++ b/src/server/api/endpoints/drive/files/find.ts @@ -31,5 +31,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { 'metadata.folderId': ps.folderId }); - res(await Promise.all(files.map(file => pack(file)))); + res(await Promise.all(files.map(file => pack(file, { self: true })))); })); diff --git a/src/server/api/endpoints/drive/files/show.ts b/src/server/api/endpoints/drive/files/show.ts index 450a97065b30b4949f144be2dc44d28adddff1e4..95c3323fbbade93a99836a17fb0789e0ad1de340 100644 --- a/src/server/api/endpoints/drive/files/show.ts +++ b/src/server/api/endpoints/drive/files/show.ts @@ -41,7 +41,8 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { // Serialize const _file = await pack(file, { - detail: true + detail: true, + self: true }); res(_file); diff --git a/src/server/api/endpoints/drive/files/update.ts b/src/server/api/endpoints/drive/files/update.ts index 4efec3dc2a56314f7b32612f80dcc191606407b5..a5835c6d65c26b9b62239de9daf53634d9b431fc 100644 --- a/src/server/api/endpoints/drive/files/update.ts +++ b/src/server/api/endpoints/drive/files/update.ts @@ -111,7 +111,7 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { }); // Serialize - const fileObj = await pack(file); + const fileObj = await pack(file, { self: true }); // Response res(fileObj); diff --git a/src/server/api/endpoints/drive/files/upload_from_url.ts b/src/server/api/endpoints/drive/files/upload_from_url.ts index b7b9cb41c40ba0cdf7d4bb4ced9975f97cee67b3..fc386e16389ab6cdb5a7648549c92adeb810ed58 100644 --- a/src/server/api/endpoints/drive/files/upload_from_url.ts +++ b/src/server/api/endpoints/drive/files/upload_from_url.ts @@ -50,5 +50,5 @@ export const meta = { }; export default define(meta, (ps, user) => new Promise(async (res, rej) => { - res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force))); + res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true })); })); diff --git a/src/server/api/endpoints/drive/stream.ts b/src/server/api/endpoints/drive/stream.ts index 804ecf50d96c52a634f9aaeacf0de08a4c88ebda..c8342c66b5d9bdde0c0a4e407a26768448c1be07 100644 --- a/src/server/api/endpoints/drive/stream.ts +++ b/src/server/api/endpoints/drive/stream.ts @@ -65,5 +65,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => { sort: sort }); - res(await packMany(files)); + res(await packMany(files, { self: true })); })); diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index b904bda91b720740a1d21bed3685070247d9f10d..c64177d4ee2741ef81ec02dd19ab24aaf6a3bcce 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -3,6 +3,7 @@ import * as send from 'koa-send'; import * as mongodb from 'mongodb'; import DriveFile, { getDriveFileBucket } from '../../models/drive-file'; import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; +import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic'; const assets = `${__dirname}/../../server/file/assets/`; @@ -41,6 +42,11 @@ export default async function(ctx: Koa.Context) { } const sendRaw = async () => { + if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) { + ctx.status = 403; + return; + } + const bucket = await getDriveFileBucket(); const readable = bucket.openDownloadStream(fileId); readable.on('error', commonReadableHandlerGenerator(ctx)); @@ -60,6 +66,19 @@ export default async function(ctx: Koa.Context) { } else { await sendRaw(); } + } else if ('web' in ctx.query) { + const web = await DriveFileWebpublic.findOne({ + 'metadata.originalId': fileId + }); + + if (web != null) { + ctx.set('Content-Type', file.contentType); + + const bucket = await getDriveFileWebpublicBucket(); + ctx.body = bucket.openDownloadStream(web._id); + } else { + await sendRaw(); + } } else { if ('download' in ctx.query) { ctx.set('Content-Disposition', 'attachment'); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index d5156de6c43e4413ce8c05cbd5cbf404515e0e87..2ea8cdc3bd6508e24c75da3f86d55c162012e66e 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -16,6 +16,7 @@ import { publishMainStream, publishDriveStream } from '../../stream'; import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import delFile from './delete-file'; import config from '../../config'; +import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic'; import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; import driveChart from '../../chart/drive'; import perUserDriveChart from '../../chart/per-user-drive'; @@ -23,7 +24,71 @@ import fetchMeta from '../../misc/fetch-meta'; const log = debug('misskey:drive:add-file'); -async function save(path: string, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> { +/*** + * Save file + * @param path Path for original + * @param name Name for original + * @param type Content-Type for original + * @param hash Hash for original + * @param size Size for original + * @param metadata + */ +async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> { + // #region webpublic + let webpublic: Buffer; + let webpublicExt = 'jpg'; + let webpublicType = 'image/jpeg'; + + if (!metadata.uri) { // from local instance + log(`creating web image`); + + if (['image/jpeg'].includes(type)) { + webpublic = await sharp(path) + .resize(2048, 2048, { + fit: 'inside', + withoutEnlargement: true + }) + .rotate() + .jpeg({ + quality: 85, + progressive: true + }) + .toBuffer(); + } else if (['image/webp'].includes(type)) { + webpublic = await sharp(path) + .resize(2048, 2048, { + fit: 'inside', + withoutEnlargement: true + }) + .rotate() + .webp({ + quality: 85 + }) + .toBuffer(); + + webpublicExt = 'webp'; + webpublicType = 'image/webp'; + } else if (['image/png'].includes(type)) { + webpublic = await sharp(path) + .resize(2048, 2048, { + fit: 'inside', + withoutEnlargement: true + }) + .rotate() + .png() + .toBuffer(); + + webpublicExt = 'png'; + webpublicType = 'image/png'; + } else { + log(`web image not created (not an image)`); + } + } else { + log(`web image not created (from remote)`); + } + // #endregion webpublic + + // #region thumbnail let thumbnail: Buffer; let thumbnailExt = 'jpg'; let thumbnailType = 'image/jpeg'; @@ -53,10 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size thumbnailExt = 'png'; thumbnailType = 'image/png'; } + // #endregion thumbnail if (config.drive && config.drive.storage == 'minio') { - const minio = new Minio.Client(config.drive.config); - let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); if (ext === '') { @@ -66,33 +130,41 @@ async function save(path: string, name: string, type: string, hash: string, size } const key = `${config.drive.prefix}/${uuid.v4()}${ext}`; + const webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${webpublicExt}`; const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`; - const baseUrl = config.drive.baseUrl - || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; + log(`uploading original: ${key}`); + const uploads = [ + upload(key, fs.createReadStream(path), type) + ]; - await minio.putObject(config.drive.bucket, key, fs.createReadStream(path), size, { - 'Content-Type': type, - 'Cache-Control': 'max-age=31536000, immutable' - }); + if (webpublic) { + log(`uploading webpublic: ${webpublicKey}`); + uploads.push(upload(webpublicKey, webpublic, webpublicType)); + } if (thumbnail) { - await minio.putObject(config.drive.bucket, thumbnailKey, thumbnail, size, { - 'Content-Type': thumbnailType, - 'Cache-Control': 'max-age=31536000, immutable' - }); + log(`uploading thumbnail: ${thumbnailKey}`); + uploads.push(upload(thumbnailKey, thumbnail, thumbnailType)); } + await Promise.all(uploads); + + const baseUrl = config.drive.baseUrl + || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; + Object.assign(metadata, { withoutChunks: true, storage: 'minio', storageProps: { key: key, - thumbnailKey: thumbnailKey + webpublicKey: webpublic ? webpublicKey : null, + thumbnailKey: thumbnail ? thumbnailKey : null, }, url: `${ baseUrl }/${ key }`, + webpublicUrl: webpublic ? `${ baseUrl }/${ webpublicKey }` : null, thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null - }); + } as IMetadata); const file = await DriveFile.insert({ length: size, @@ -105,29 +177,55 @@ async function save(path: string, name: string, type: string, hash: string, size return file; } else { - // Get MongoDB GridFS bucket - const bucket = await getDriveFileBucket(); + // #region store original + const originalDst = await getDriveFileBucket(); - const file = await new Promise<IDriveFile>((resolve, reject) => { - const writeStream = bucket.openUploadStream(name, { + // web用(Exif削除済ã¿)ãŒã‚ã‚‹å ´åˆã¯ã‚ªãƒªã‚¸ãƒŠãƒ«ã«ã‚¢ã‚¯ã‚»ã‚¹åˆ¶é™ + if (webpublic) metadata.accessKey = uuid.v4(); + + const originalFile = await new Promise<IDriveFile>((resolve, reject) => { + const writeStream = originalDst.openUploadStream(name, { contentType: type, metadata }); writeStream.once('finish', resolve); writeStream.on('error', reject); - fs.createReadStream(path).pipe(writeStream); }); + log(`original stored to ${originalFile._id}`); + // #endregion store original + + // #region store webpublic + if (webpublic) { + const webDst = await getDriveFileWebpublicBucket(); + + const webFile = await new Promise<IDriveFile>((resolve, reject) => { + const writeStream = webDst.openUploadStream(name, { + contentType: webpublicType, + metadata: { + originalId: originalFile._id + } + }); + + writeStream.once('finish', resolve); + writeStream.on('error', reject); + writeStream.end(webpublic); + }); + + log(`web stored ${webFile._id}`); + } + // #endregion store webpublic + if (thumbnail) { const thumbnailBucket = await getDriveFileThumbnailBucket(); - await new Promise<IDriveFile>((resolve, reject) => { + const tuhmFile = await new Promise<IDriveFile>((resolve, reject) => { const writeStream = thumbnailBucket.openUploadStream(name, { contentType: thumbnailType, metadata: { - originalId: file._id + originalId: originalFile._id } }); @@ -135,12 +233,23 @@ async function save(path: string, name: string, type: string, hash: string, size writeStream.on('error', reject); writeStream.end(thumbnail); }); + + log(`thumbnail stored ${tuhmFile._id}`); } - return file; + return originalFile; } } +async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) { + const minio = new Minio.Client(config.drive.config); + + await minio.putObject(config.drive.bucket, key, stream, null, { + 'Content-Type': type, + 'Cache-Control': 'max-age=31536000, immutable' + }); +} + async function deleteOldFile(user: IRemoteUser) { const oldFile = await DriveFile.findOne({ _id: { diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 3e2f42003b352ab7841d6556b0506ef1dac6e577..92d0010bcf225eb3a4f2dccb21f3f927909ed587 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -4,6 +4,7 @@ import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive- import config from '../../config'; import driveChart from '../../chart/drive'; import perUserDriveChart from '../../chart/per-user-drive'; +import DriveFileWebpublic, { DriveFileWebpublicChunk } from '../../models/drive-file-webpublic'; export default async function(file: IDriveFile, isExpired = false) { if (file.metadata.storage == 'minio') { @@ -20,6 +21,11 @@ export default async function(file: IDriveFile, isExpired = false) { const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`; await minio.removeObject(config.drive.bucket, thumbnailObj); } + + if (file.metadata.webpublicUrl) { + const webpublicObj = file.metadata.storageProps.webpublicKey ? file.metadata.storageProps.webpublicKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-original`; + await minio.removeObject(config.drive.bucket, webpublicObj); + } } // ãƒãƒ£ãƒ³ã‚¯ã‚’ã™ã¹ã¦å‰Šé™¤ @@ -48,6 +54,20 @@ export default async function(file: IDriveFile, isExpired = false) { } //#endregion + //#region Web公開用もã‚ã‚Œã°å‰Šé™¤ + const webpublic = await DriveFileWebpublic.findOne({ + 'metadata.originalId': file._id + }); + + if (webpublic) { + await DriveFileWebpublicChunk.remove({ + files_id: webpublic._id + }); + + await DriveFileWebpublic.remove({ _id: webpublic._id }); + } + //#endregion + // 統計を更新 driveChart.update(file, false); perUserDriveChart.update(file, false);