From 0dead4637e650285bbec75f4d541f0ab76f2a116 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 00:21:44 +0100 Subject: [PATCH 01/10] add: bunnycdn storage support --- packages/backend/src/core/BunnyService.ts | 68 +++++++++++++++++++++++ packages/backend/src/core/CoreModule.ts | 6 ++ packages/backend/src/core/DriveService.ts | 41 ++++++++------ 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 packages/backend/src/core/BunnyService.ts diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts new file mode 100644 index 0000000000..d7d174aa2e --- /dev/null +++ b/packages/backend/src/core/BunnyService.ts @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: marie and sharkey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as https from 'node:https'; +import * as fs from 'node:fs'; +import { Readable } from 'node:stream'; +import { finished } from 'node:stream/promises'; +import { Injectable } from '@nestjs/common'; +import type { MiMeta } from '@/models/Meta.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class BunnyService { + constructor( + private httpRequestService: HttpRequestService, + ) { + } + + @bindThis + public getBunnyInfo(meta: MiMeta) { + return { + endpoint: meta.objectStorageEndpoint ?? undefined, + accessKey: meta.objectStorageSecretKey ?? '', + zone: meta.objectStorageBucket ?? undefined, + prefix: meta.objectStoragePrefix ?? '', + }; + } + + @bindThis + public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { + const client = this.getBunnyInfo(meta); + + // Required to convert the buffer from webpulic and thumbnail to a ReadableStream for PUT + const data = Buffer.isBuffer(input) ? Readable.from(input) : input; + + const options = { + method: 'PUT', + host: client.endpoint, + path: `/${client.zone}/${path}`, + headers: { + AccessKey: client.accessKey, + 'Content-Type': 'application/octet-stream', + }, + }; + + const req = https.request(options); + + req.on('error', (error) => { + console.error(error); + }); + + data.pipe(req).on('finish', () => { + data.destroy(); + }); + + // wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success wait too early + await finished(data); + } + + @bindThis + public delete(meta: MiMeta, file: string) { + const client = this.getBunnyInfo(meta); + return this.httpRequestService.send(`https://${client.endpoint}/${client.zone}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } }); + } +} diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 997d81facc..b12703138d 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -61,6 +61,7 @@ import { ReactionsBufferingService } from './ReactionsBufferingService.js'; import { RelayService } from './RelayService.js'; import { RoleService } from './RoleService.js'; import { S3Service } from './S3Service.js'; +import { BunnyService } from './BunnyService.js'; import { SignupService } from './SignupService.js'; import { WebAuthnService } from './WebAuthnService.js'; import { UserBlockingService } from './UserBlockingService.js'; @@ -208,6 +209,7 @@ const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingServi const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService }; const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService }; const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service }; +const $BunnyService: Provider = { provide: 'BunnyService', useExisting: BunnyService }; const $SignupService: Provider = { provide: 'SignupService', useExisting: SignupService }; const $WebAuthnService: Provider = { provide: 'WebAuthnService', useExisting: WebAuthnService }; const $UserBlockingService: Provider = { provide: 'UserBlockingService', useExisting: UserBlockingService }; @@ -367,6 +369,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp RelayService, RoleService, S3Service, + BunnyService, SignupService, WebAuthnService, UserBlockingService, @@ -522,6 +525,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $RelayService, $RoleService, $S3Service, + $BunnyService, $SignupService, $WebAuthnService, $UserBlockingService, @@ -678,6 +682,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp RelayService, RoleService, S3Service, + BunnyService, SignupService, WebAuthnService, UserBlockingService, @@ -832,6 +837,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $RelayService, $RoleService, $S3Service, + $BunnyService, $SignupService, $WebAuthnService, $UserBlockingService, diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index a65059b417..3d53a94b43 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -44,6 +44,7 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { BunnyService } from '@/core/BunnyService.js'; type AddFileArgs = { /** User who wish to add file */ @@ -121,6 +122,7 @@ export class DriveService { private downloadService: DownloadService, private internalStorageService: InternalStorageService, private s3Service: S3Service, + private bunnyService: BunnyService, private imageProcessingService: ImageProcessingService, private videoProcessingService: VideoProcessingService, private globalEventService: GlobalEventService, @@ -405,20 +407,24 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - await this.s3Service.upload(this.meta, params) - .then( - result => { - if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput - this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); - } else { // AbortMultipartUploadCommandOutput - this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); - } - }) - .catch( - err => { - this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); - }, - ); + if (this.meta.objectStorageAccessKey) { + await this.s3Service.upload(this.meta, params) + .then( + result => { + if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput + this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); + } else { // AbortMultipartUploadCommandOutput + this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); + } + }) + .catch( + err => { + this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); + }, + ); + } else { + await this.bunnyService.upload(this.meta, key, stream); + } } // Expire oldest file (without avatar or banner) of remote user @@ -814,8 +820,11 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - - await this.s3Service.delete(this.meta, param); + if (this.meta.objectStorageAccessKey) { + await this.s3Service.delete(this.meta, param); + } else { + await this.bunnyService.delete(this.meta, key); + } } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); -- GitLab From 97fb73ca2ce043e042b930445e4a93aa9beff65d Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 00:33:13 +0100 Subject: [PATCH 02/10] fix: test failing due to undefined --- packages/backend/src/core/BunnyService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index d7d174aa2e..3e0b37da7c 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -22,9 +22,9 @@ export class BunnyService { @bindThis public getBunnyInfo(meta: MiMeta) { return { - endpoint: meta.objectStorageEndpoint ?? undefined, + endpoint: meta.objectStorageEndpoint ?? 'example.net', accessKey: meta.objectStorageSecretKey ?? '', - zone: meta.objectStorageBucket ?? undefined, + zone: meta.objectStorageBucket ?? '', prefix: meta.objectStoragePrefix ?? '', }; } -- GitLab From 3bdac95bfd95c8f0845b4ea80ed98cd820833c06 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 00:41:37 +0100 Subject: [PATCH 03/10] upd: check if endpoint includes bunnycdn.com --- packages/backend/src/core/DriveService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 3d53a94b43..83791f7e8d 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageAccessKey) { + if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.upload(this.meta, params) .then( result => { @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageAccessKey) { + if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.delete(this.meta, param); } else { await this.bunnyService.delete(this.meta, key); -- GitLab From 6dae5c916599d595215c82a22a120e998a77fd11 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 00:42:28 +0100 Subject: [PATCH 04/10] upd: remove old check --- packages/backend/src/core/DriveService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 83791f7e8d..616deb0221 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.upload(this.meta, params) .then( result => { @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.delete(this.meta, param); } else { await this.bunnyService.delete(this.meta, key); -- GitLab From a35bfa9f1a4f426c1c7bdfead9f7554d11f31427 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 00:46:23 +0100 Subject: [PATCH 05/10] upd flip check --- packages/backend/src/core/DriveService.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 616deb0221..53bc4e553f 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,9 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + await this.bunnyService.upload(this.meta, key, stream); + } else { await this.s3Service.upload(this.meta, params) .then( result => { @@ -422,8 +424,6 @@ export class DriveService { this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); }, ); - } else { - await this.bunnyService.upload(this.meta, key, stream); } } @@ -820,10 +820,10 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { - await this.s3Service.delete(this.meta, param); - } else { + if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { await this.bunnyService.delete(this.meta, key); + } else { + await this.s3Service.delete(this.meta, param); } } catch (err: any) { if (err.name === 'NoSuchKey') { -- GitLab From 137cb430b52cbcd755f5fdbc3b4cc854323b7abe Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 05:40:59 +0100 Subject: [PATCH 06/10] upd: remove default endpoint value --- packages/backend/src/core/BunnyService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 3e0b37da7c..5d7c621cb2 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -22,7 +22,7 @@ export class BunnyService { @bindThis public getBunnyInfo(meta: MiMeta) { return { - endpoint: meta.objectStorageEndpoint ?? 'example.net', + endpoint: meta.objectStorageEndpoint ?? '', accessKey: meta.objectStorageSecretKey ?? '', zone: meta.objectStorageBucket ?? '', prefix: meta.objectStoragePrefix ?? '', -- GitLab From 76b0c1ce26cda27f73e49181101b449f5d9d600c Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 14:31:41 +0100 Subject: [PATCH 07/10] Apply suggestions --- packages/backend/src/core/BunnyService.ts | 25 ++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 5d7c621cb2..1563bc2711 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -22,10 +22,10 @@ export class BunnyService { @bindThis public getBunnyInfo(meta: MiMeta) { return { - endpoint: meta.objectStorageEndpoint ?? '', - accessKey: meta.objectStorageSecretKey ?? '', - zone: meta.objectStorageBucket ?? '', - prefix: meta.objectStoragePrefix ?? '', + endpoint: meta.objectStorageEndpoint, + accessKey: meta.objectStorageSecretKey, + zone: meta.objectStorageBucket, + fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`, }; } @@ -33,9 +33,16 @@ export class BunnyService { public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { const client = this.getBunnyInfo(meta); - // Required to convert the buffer from webpulic and thumbnail to a ReadableStream for PUT + if (!client.endpoint || !client.zone || !client.accessKey) { + return console.error('Missing Information'); + } + + // Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT const data = Buffer.isBuffer(input) ? Readable.from(input) : input; + const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true); + + // Seperation of path and host/domain is required here const options = { method: 'PUT', host: client.endpoint, @@ -44,6 +51,7 @@ export class BunnyService { AccessKey: client.accessKey, 'Content-Type': 'application/octet-stream', }, + agent: agent, }; const req = https.request(options); @@ -56,13 +64,16 @@ export class BunnyService { data.destroy(); }); - // wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success wait too early + // wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early await finished(data); } @bindThis public delete(meta: MiMeta, file: string) { const client = this.getBunnyInfo(meta); - return this.httpRequestService.send(`https://${client.endpoint}/${client.zone}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } }); + if (!client.endpoint || !client.zone || !client.accessKey) { + return; + } + return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } }); } } -- GitLab From 0481b25a62457f9ae44f1ef60e71dbab1944bde1 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 14:36:44 +0100 Subject: [PATCH 08/10] upd: create usingBunnyCDN --- packages/backend/src/core/BunnyService.ts | 7 ++++++- packages/backend/src/core/DriveService.ts | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 1563bc2711..3931ccfdd1 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -29,6 +29,11 @@ export class BunnyService { }; } + @bindThis + public usingBunnyCDN(meta: MiMeta) { + return meta.objectStorageEndpoint && meta.objectStorageEndpoint.includes('bunnycdn.com') ? true : false; + } + @bindThis public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { const client = this.getBunnyInfo(meta); @@ -41,7 +46,7 @@ export class BunnyService { const data = Buffer.isBuffer(input) ? Readable.from(input) : input; const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true); - + // Seperation of path and host/domain is required here const options = { method: 'PUT', diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 53bc4e553f..31c065d4b4 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + if (this.bunnyService.usingBunnyCDN(this.meta)) { await this.bunnyService.upload(this.meta, key, stream); } else { await this.s3Service.upload(this.meta, params) @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + if (this.bunnyService.usingBunnyCDN(this.meta)) { await this.bunnyService.delete(this.meta, key); } else { await this.s3Service.delete(this.meta, param); -- GitLab From 003c37e84cd0cb5de595db178d8b0da65de30a73 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 19:42:25 +0100 Subject: [PATCH 09/10] upd: add comment, throw on missing details --- packages/backend/src/core/BunnyService.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 3931ccfdd1..4787954eef 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -21,8 +21,16 @@ export class BunnyService { @bindThis public getBunnyInfo(meta: MiMeta) { + if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) { + throw new Error('One of the following fields is empty: Endpoint, Bucket, Secret Key'); + } + return { endpoint: meta.objectStorageEndpoint, + /* + The way S3 works is that the Secret Key is essentially the password for the API but Bunny calls their password AccessKey so we call it accessKey here. + Bunny also doesn't specify a username/s3 access key when doing HTTP API requests so we end up not using our Access Key field from the form. + */ accessKey: meta.objectStorageSecretKey, zone: meta.objectStorageBucket, fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`, @@ -38,10 +46,6 @@ export class BunnyService { public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { const client = this.getBunnyInfo(meta); - if (!client.endpoint || !client.zone || !client.accessKey) { - return console.error('Missing Information'); - } - // Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT const data = Buffer.isBuffer(input) ? Readable.from(input) : input; @@ -76,9 +80,6 @@ export class BunnyService { @bindThis public delete(meta: MiMeta, file: string) { const client = this.getBunnyInfo(meta); - if (!client.endpoint || !client.zone || !client.accessKey) { - return; - } return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } }); } } -- GitLab From 381ea14049bd0d351b1326f01c6e5f9b2cb69216 Mon Sep 17 00:00:00 2001 From: Marie <github@yuugi.dev> Date: Fri, 28 Mar 2025 20:16:34 +0100 Subject: [PATCH 10/10] upd: throw indetifiableerror instead of normal error --- packages/backend/src/core/BunnyService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 4787954eef..09212fffbf 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -11,6 +11,7 @@ import { Injectable } from '@nestjs/common'; import type { MiMeta } from '@/models/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; @Injectable() export class BunnyService { @@ -22,7 +23,7 @@ export class BunnyService { @bindThis public getBunnyInfo(meta: MiMeta) { if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) { - throw new Error('One of the following fields is empty: Endpoint, Bucket, Secret Key'); + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf90', 'Failed to use BunnyCDN, One of the required fields is missing.'); } return { -- GitLab