From 88e3d3e8cbe26c0280308e819965a64a91491f90 Mon Sep 17 00:00:00 2001
From: tamaina <tamaina@hotmail.co.jp>
Date: Sat, 11 Mar 2023 14:11:40 +0900
Subject: [PATCH] =?UTF-8?q?enhance(server):=20=E7=94=BB=E5=83=8F=E5=9C=A7?=
=?UTF-8?q?=E7=B8=AE=E5=91=A8=E3=82=8A=EF=BC=88=E4=B8=BB=E3=81=AB=E3=82=B5?=
=?UTF-8?q?=E3=83=A0=E3=83=8D=E3=82=A4=E3=83=AB=E3=81=AE=E4=BB=95=E6=A7=98?=
=?UTF-8?q?=EF=BC=89=E3=81=AE=E5=A4=89=E6=9B=B4=20(#10287)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* DriveService, is-mime-image
* static, previewã‚’avifã«, アニメーション画åƒã§ã‚‚thumbnailを生æˆ
* fallback
* animated: true
* fix
* avatarã¯webp
* revert ?? file.url
---------
Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>
---
packages/backend/src/core/DriveService.ts | 37 ++++----
.../src/core/ImageProcessingService.ts | 91 +++++++++++--------
.../core/entities/DriveFileEntityService.ts | 4 +-
packages/backend/src/misc/is-mime-image.ts | 8 +-
.../backend/src/server/FileServerService.ts | 10 +-
.../src/server/web/UrlPreviewService.ts | 2 +-
.../frontend/src/components/MkMediaImage.vue | 2 +-
.../frontend/src/pages/user/index.photos.vue | 2 +-
packages/frontend/src/scripts/media-proxy.ts | 7 +-
.../frontend/src/widgets/WidgetPhotos.vue | 2 +-
10 files changed, 91 insertions(+), 74 deletions(-)
diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index dfaacffc1d..7eccf4b3b1 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -2,6 +2,7 @@ import * as fs from 'node:fs';
import { Inject, Injectable } from '@nestjs/common';
import { v4 as uuid } from 'uuid';
import sharp from 'sharp';
+import { sharpBmp } from 'sharp-read-bmp';
import { IsNull } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
@@ -34,6 +35,7 @@ import { FileInfoService } from '@/core/FileInfoService.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { correctFilename } from '@/misc/correct-filename.js';
+import { isMimeImage } from '@/misc/is-mime-image.js';
import type S3 from 'aws-sdk/clients/s3.js';
type AddFileArgs = {
@@ -274,8 +276,8 @@ export class DriveService {
}
}
- if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) {
- this.registerLogger.debug('web image and thumbnail not created (not an required file)');
+ if (!isMimeImage(type, 'sharp-convertible-image-with-bmp')) {
+ this.registerLogger.debug('web image and thumbnail not created (cannot convert by sharp)');
return {
webpublic: null,
thumbnail: null,
@@ -284,22 +286,16 @@ export class DriveService {
let img: sharp.Sharp | null = null;
let satisfyWebpublic: boolean;
+ let isAnimated: boolean;
try {
- img = sharp(path);
+ img = await sharpBmp(path, type);
const metadata = await img.metadata();
- const isAnimated = metadata.pages && metadata.pages > 1;
-
- // skip animated
- if (isAnimated) {
- return {
- webpublic: null,
- thumbnail: null,
- };
- }
+ isAnimated = !!(metadata.pages && metadata.pages > 1);
satisfyWebpublic = !!(
- type !== 'image/svg+xml' && type !== 'image/avif' &&
+ type !== 'image/svg+xml' && // security reason
+ type !== 'image/avif' && // not supported by Mastodon
!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
metadata.width && metadata.width <= 2048 &&
metadata.height && metadata.height <= 2048
@@ -315,15 +311,13 @@ export class DriveService {
// #region webpublic
let webpublic: IImage | null = null;
- if (generateWeb && !satisfyWebpublic) {
+ if (generateWeb && !satisfyWebpublic && !isAnimated) {
this.registerLogger.info('creating web image');
try {
- if (type === 'image/jpeg') {
- webpublic = await this.imageProcessingService.convertSharpToJpeg(img, 2048, 2048);
- } else if (['image/webp', 'image/avif'].includes(type)) {
+ if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
webpublic = await this.imageProcessingService.convertSharpToWebp(img, 2048, 2048);
- } else if (['image/png', 'image/svg+xml'].includes(type)) {
+ } else if (['image/png', 'image/bmp', 'image/svg+xml'].includes(type)) {
webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
} else {
this.registerLogger.debug('web image not created (not an required image)');
@@ -333,6 +327,7 @@ export class DriveService {
}
} else {
if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)');
+ else if (isAnimated) this.registerLogger.info('web image not created (animated image)');
else this.registerLogger.info('web image not created (from remote)');
}
// #endregion webpublic
@@ -341,10 +336,10 @@ export class DriveService {
let thumbnail: IImage | null = null;
try {
- if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) {
- thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 280);
+ if (isAnimated) {
+ thumbnail = await this.imageProcessingService.convertSharpToWebp(sharp(path, { animated: true }), 374, 317, { alphaQuality: 70 });
} else {
- this.registerLogger.debug('thumbnail not created (not an required file)');
+ thumbnail = await this.imageProcessingService.convertSharpToAvif(img, 498, 422);
}
} catch (err) {
this.registerLogger.warn('thumbnail not created (an error occured)', err as Error);
diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts
index 7c88f5e9a0..3246475d12 100644
--- a/packages/backend/src/core/ImageProcessingService.ts
+++ b/packages/backend/src/core/ImageProcessingService.ts
@@ -15,15 +15,28 @@ export type IImageStream = {
type: string;
};
-export type IImageStreamable = IImage | IImageStream;
+export type IImageSharp = {
+ data: sharp.Sharp;
+ ext: string | null;
+ type: string;
+};
+
+export type IImageStreamable = IImage | IImageStream | IImageSharp;
export const webpDefault: sharp.WebpOptions = {
- quality: 85,
+ quality: 77,
alphaQuality: 95,
lossless: false,
nearLossless: false,
smartSubsample: true,
mixed: true,
+ effort: 2,
+};
+
+export const avifDefault: sharp.AvifOptions = {
+ quality: 60,
+ lossless: false,
+ effort: 2,
};
import { bindThis } from '@/decorators.js';
@@ -38,90 +51,96 @@ export class ImageProcessingService {
}
/**
- * Convert to JPEG
+ * Convert to WebP
* with resize, remove metadata, resolve orientation, stop animation
*/
@bindThis
- public async convertToJpeg(path: string, width: number, height: number): Promise<IImage> {
- return this.convertSharpToJpeg(await sharp(path), width, height);
+ public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
+ return this.convertSharpToWebp(sharp(path), width, height, options);
}
@bindThis
- public async convertSharpToJpeg(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
- const data = await sharp
+ public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
+ const result = this.convertSharpToWebpStream(sharp, width, height, options);
+
+ return {
+ data: await result.data.toBuffer(),
+ ext: result.ext,
+ type: result.type,
+ };
+ }
+
+ @bindThis
+ public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp {
+ return this.convertSharpToWebpStream(sharp(path), width, height, options);
+ }
+
+ @bindThis
+ public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp {
+ const data = sharp
.resize(width, height, {
fit: 'inside',
withoutEnlargement: true,
})
.rotate()
- .jpeg({
- quality: 85,
- progressive: true,
- })
- .toBuffer();
+ .webp(options);
return {
data,
- ext: 'jpg',
- type: 'image/jpeg',
+ ext: 'webp',
+ type: 'image/webp',
};
}
/**
- * Convert to WebP
+ * Convert to Avif
* with resize, remove metadata, resolve orientation, stop animation
*/
@bindThis
- public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
- return this.convertSharpToWebp(sharp(path), width, height, options);
+ public async convertToAvif(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise<IImage> {
+ return this.convertSharpToAvif(sharp(path), width, height, options);
}
@bindThis
- public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
- const data = await sharp
- .resize(width, height, {
- fit: 'inside',
- withoutEnlargement: true,
- })
- .rotate()
- .webp(options)
- .toBuffer();
+ public async convertSharpToAvif(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise<IImage> {
+ const result = this.convertSharpToAvifStream(sharp, width, height, options);
return {
- data,
- ext: 'webp',
- type: 'image/webp',
+ data: await result.data.toBuffer(),
+ ext: result.ext,
+ type: result.type,
};
}
@bindThis
- public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream {
- return this.convertSharpToWebpStream(sharp(path), width, height, options);
+ public convertToAvifStream(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp {
+ return this.convertSharpToAvifStream(sharp(path), width, height, options);
}
@bindThis
- public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream {
+ public convertSharpToAvifStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp {
const data = sharp
.resize(width, height, {
fit: 'inside',
withoutEnlargement: true,
})
.rotate()
- .webp(options);
+ .avif(options);
return {
data,
- ext: 'webp',
- type: 'image/webp',
+ ext: 'avif',
+ type: 'image/avif',
};
}
+
/**
* Convert to PNG
* with resize, remove metadata, resolve orientation, stop animation
*/
@bindThis
public async convertToPng(path: string, width: number, height: number): Promise<IImage> {
- return this.convertSharpToPng(await sharp(path), width, height);
+ return this.convertSharpToPng(sharp(path), width, height);
}
@bindThis
diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts
index 74a0689d89..1a6913b800 100644
--- a/packages/backend/src/core/entities/DriveFileEntityService.ts
+++ b/packages/backend/src/core/entities/DriveFileEntityService.ts
@@ -76,7 +76,7 @@ export class DriveFileEntityService {
@bindThis
private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string {
return appendQuery(
- `${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
+ `${this.config.mediaProxy}/${mode ?? 'image'}.${mode === 'avatar' ? 'webp' : 'avif'}`,
query({
url,
...(mode ? { [mode]: '1' } : {}),
@@ -104,7 +104,7 @@ export class DriveFileEntityService {
const url = file.webpublicUrl ?? file.url;
- return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null);
+ return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? url : null);
}
@bindThis
diff --git a/packages/backend/src/misc/is-mime-image.ts b/packages/backend/src/misc/is-mime-image.ts
index 0b6d147dc1..46a66efc0f 100644
--- a/packages/backend/src/misc/is-mime-image.ts
+++ b/packages/backend/src/misc/is-mime-image.ts
@@ -2,10 +2,10 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
const dictionary = {
'safe-file': FILE_TYPE_BROWSERSAFE,
- 'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
- 'sharp-animation-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'],
- 'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
- 'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
+ 'sharp-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
+ 'sharp-animation-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'],
+ 'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
+ 'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'],
};
export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);
diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts
index 6db9a9672c..fb1c67f20d 100644
--- a/packages/backend/src/server/FileServerService.ts
+++ b/packages/backend/src/server/FileServerService.ts
@@ -130,7 +130,7 @@ export class FileServerService {
if (isMimeImage(file.mime, 'sharp-convertible-image-with-bmp')) {
reply.header('Cache-Control', 'max-age=31536000, immutable');
- const url = new URL(`${this.config.mediaProxy}/static.webp`);
+ const url = new URL(`${this.config.mediaProxy}/static.avif`);
url.searchParams.set('url', file.url);
url.searchParams.set('static', '1');
@@ -151,7 +151,7 @@ export class FileServerService {
if (['image/svg+xml'].includes(file.mime)) {
reply.header('Cache-Control', 'max-age=31536000, immutable');
- const url = new URL(`${this.config.mediaProxy}/svg.webp`);
+ const url = new URL(`${this.config.mediaProxy}/svg.avif`);
url.searchParams.set('url', file.url);
file.cleanup();
@@ -291,9 +291,9 @@ export class FileServerService {
};
}
} else if ('static' in request.query) {
- image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 498, 280);
+ image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 498, 422);
} else if ('preview' in request.query) {
- image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 200, 200);
+ image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 200, 200);
} else if ('badge' in request.query) {
const mask = (await sharpBmp(file.path, file.mime))
.resize(96, 96, {
@@ -325,7 +325,7 @@ export class FileServerService {
type: 'image/png',
};
} else if (file.mime === 'image/svg+xml') {
- image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048);
+ image = this.imageProcessingService.convertToAvifStream(file.path, 2048, 2048);
} else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) {
throw new StatusError('Rejected type', 403, 'Rejected type');
}
diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts
index 2ce7293a52..5f4d53d0ec 100644
--- a/packages/backend/src/server/web/UrlPreviewService.ts
+++ b/packages/backend/src/server/web/UrlPreviewService.ts
@@ -33,7 +33,7 @@ export class UrlPreviewService {
private wrap(url?: string | null): string | null {
return url != null
? url.match(/^https?:\/\//)
- ? `${this.config.mediaProxy}/preview.webp?${query({
+ ? `${this.config.mediaProxy}/preview.avif?${query({
url,
preview: '1',
})}`
diff --git a/packages/frontend/src/components/MkMediaImage.vue b/packages/frontend/src/components/MkMediaImage.vue
index 6091b40016..a4065dcd07 100644
--- a/packages/frontend/src/components/MkMediaImage.vue
+++ b/packages/frontend/src/components/MkMediaImage.vue
@@ -43,7 +43,7 @@ let darkMode = $ref(defaultStore.state.darkMode);
const url = (props.raw || defaultStore.state.loadRawImages)
? props.image.url
: defaultStore.state.disableShowingAnimatedImages
- ? getStaticImageUrl(props.image.thumbnailUrl)
+ ? getStaticImageUrl(props.image.url)
: props.image.thumbnailUrl;
// Plugin:register_note_view_interruptor を使ã£ã¦æ›¸ãæ›ãˆã‚‰ã‚Œã‚‹å¯èƒ½æ€§ãŒã‚ã‚‹ãŸã‚watchã™ã‚‹
diff --git a/packages/frontend/src/pages/user/index.photos.vue b/packages/frontend/src/pages/user/index.photos.vue
index 607082c1e4..85f6591eee 100644
--- a/packages/frontend/src/pages/user/index.photos.vue
+++ b/packages/frontend/src/pages/user/index.photos.vue
@@ -41,7 +41,7 @@ let images = $ref<{
function thumbnail(image: misskey.entities.DriveFile): string {
return defaultStore.state.disableShowingAnimatedImages
- ? getStaticImageUrl(image.thumbnailUrl)
+ ? getStaticImageUrl(image.url)
: image.thumbnailUrl;
}
diff --git a/packages/frontend/src/scripts/media-proxy.ts b/packages/frontend/src/scripts/media-proxy.ts
index 2fe5bdcf8f..d0c95e0b75 100644
--- a/packages/frontend/src/scripts/media-proxy.ts
+++ b/packages/frontend/src/scripts/media-proxy.ts
@@ -10,7 +10,10 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview', mustOrigi
imageUrl = (new URL(imageUrl)).searchParams.get('url') ?? imageUrl;
}
- return `${mustOrigin ? localProxy : instance.mediaProxy}/image.webp?${query({
+ return `${mustOrigin ? localProxy : instance.mediaProxy}/${
+ type === 'preview' ? 'preview.avif'
+ : 'image.webp'
+ }?${query({
url: imageUrl,
fallback: '1',
...(type ? { [type]: '1' } : {}),
@@ -38,7 +41,7 @@ export function getStaticImageUrl(baseUrl: string): string {
return u.href;
}
- return `${instance.mediaProxy}/static.webp?${query({
+ return `${instance.mediaProxy}/static.avif?${query({
url: u.href,
static: '1',
})}`;
diff --git a/packages/frontend/src/widgets/WidgetPhotos.vue b/packages/frontend/src/widgets/WidgetPhotos.vue
index 562249f094..716bbb4274 100644
--- a/packages/frontend/src/widgets/WidgetPhotos.vue
+++ b/packages/frontend/src/widgets/WidgetPhotos.vue
@@ -67,7 +67,7 @@ const onDriveFileCreated = (file) => {
const thumbnail = (image: any): string => {
return defaultStore.state.disableShowingAnimatedImages
- ? getStaticImageUrl(image.thumbnailUrl)
+ ? getStaticImageUrl(image.url)
: image.thumbnailUrl;
};
--
GitLab