diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 0f5ba9696d22ff6cf159a5df56f81a49af2d0572..ad9ae4fd9adfbd67903c7f5e721d62bbd2a6a86d 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -114,9 +114,27 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌───────────────────────────┠-#───┘ MeiliSearch configuration └───────────────────────────── - +# ┌───────────────────────────────┠+#───┘ Fulltext search configuration └───────────────────────────── + +# These are the setting items for the full-text search provider. +fulltextSearch: + # You can select the ID generation method. + # - sqlLike (default) + # Use SQL-like search. + # This is a standard feature of PostgreSQL, so no special extensions are required. + # - sqlPgroonga + # Use pgroonga. + # You need to install pgroonga and configure it as a PostgreSQL extension. + # In addition to the above, you need to create a pgroonga index on the text column of the note table. + # see: https://pgroonga.github.io/tutorial/ + # - meilisearch + # Use Meilisearch. + # You need to install Meilisearch and configure. + provider: sqlLike + +# For Meilisearch settings. +# If you select "meilisearch" for "fulltextSearch.provider", it must be set. # You can set scope to local (default value) or global # (include notes from remote). diff --git a/.config/example.yml b/.config/example.yml index ea29cedd10e5b4f8e2ef4c713b660c1927626d28..349c2e9730e6c3d9cf05f46ad76e85a70040f044 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -196,9 +196,27 @@ redis: # # You can specify more ioredis options... # #username: example-username -# ┌───────────────────────────┠-#───┘ MeiliSearch configuration └───────────────────────────── - +# ┌───────────────────────────────┠+#───┘ Fulltext search configuration └───────────────────────────── + +# These are the setting items for the full-text search provider. +fulltextSearch: + # You can select the ID generation method. + # - sqlLike (default) + # Use SQL-like search. + # This is a standard feature of PostgreSQL, so no special extensions are required. + # - sqlPgroonga + # Use pgroonga. + # You need to install pgroonga and configure it as a PostgreSQL extension. + # In addition to the above, you need to create a pgroonga index on the text column of the note table. + # see: https://pgroonga.github.io/tutorial/ + # - meilisearch + # Use Meilisearch. + # You need to install Meilisearch and configure. + provider: sqlLike + +# For Meilisearch settings. +# If you select "meilisearch" for "fulltextSearch.provider", it must be set. # You can set scope to local (default value) or global # (include notes from remote). diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f60459a9df42234af8ed5c1c98df91e0b6fe0e3..e44c15005e06c2329fee6a7071cef09672a5f26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## 2025.1.0 +### Note +- [é‡è¦] ノート検索プãƒãƒã‚¤ãƒ€ã®è¿½åŠ ã«ä¼´ã„ã€configファイル(default.ymlãªã©ï¼‰ã®æ§‹æˆãŒå°‘ã—変ã‚ã‚Šã¾ã™. + - æ–°ã—ã„è¨å®šé …ç›®"fulltextSearch.provider"ãŒè¿½åŠ ã•ã‚Œã¾ã—ãŸ. sqlLike, sqlPgroonga, meilisearchã®ã„ãšã‚Œã‹ã‚’è¨å®šå‡ºæ¥ã¾ã™. + - ã™ã§ã«Meilisearchã‚’ãŠä½¿ã„ã®å ´åˆã€ **"fulltextSearch.provider"ã‚’"meilisearch"ã«è¨å®šã™ã‚‹å¿…è¦** ãŒã‚ã‚Šã¾ã™. + - 詳細㯠#14730 ãŠã‚ˆã³ `.config/example.yml` ã¾ãŸã¯ `.config/docker_example.yml`ã®'Fulltext search configuration'ã‚’ã”å‚照願ã„ã¾ã™. + ### General - @@ -30,6 +36,7 @@ ### Server - Enhance: pg_bigmãŒåˆ©ç”¨ã§ãるよã†ã€ãƒŽãƒ¼ãƒˆã®æ¤œç´¢ã‚’ILIKE演算åã§ãªãLIKE演算åã§LOWER()ã‚’ã‹ã‘ãŸãƒ†ã‚ストã«å¯¾ã—ã¦è¡Œã†ã‚ˆã†ã« +- Enhance: ノート検索ã®é¸æŠžè‚¢ã¨ã—ã¦pgroongaã«å¯¾å¿œ ( #14730 ) - Enhance: ãƒãƒ£ãƒ¼ãƒˆæ›´æ–°æ™‚ã«DBã«åŒæ™‚接続ã—ãªã„よã†ã« (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/830) - Enhance: config(default.yml)ã‹ã‚‰SQLãƒã‚°å…¨æ–‡ã‚’出力ã™ã‚‹ã‹å¦ã‹ã‚’è¨å®šå¯èƒ½ã« ( #15266 ) diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 6ae8ccfbb32ca49338e608c5bd16acdd0baf5965..ace7f7841c03a8f8ae4b1c64a52e0d9e8b4daf98 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; +import { MiMeta } from '@/models/Meta.js'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; import { RepositoryModule } from './models/RepositoryModule.js'; import { allSettled } from './misc/promise-tracker.js'; -import type { Provider, OnApplicationShutdown } from '@nestjs/common'; -import { MiMeta } from '@/models/Meta.js'; import { GlobalEvents } from './core/GlobalEventService.js'; +import type { Provider, OnApplicationShutdown } from '@nestjs/common'; const $config: Provider = { provide: DI.config, @@ -33,7 +33,11 @@ const $db: Provider = { const $meilisearch: Provider = { provide: DI.meilisearch, useFactory: (config: Config) => { - if (config.meilisearch) { + if (config.fulltextSearch?.provider === 'meilisearch') { + if (!config.meilisearch) { + throw new Error('MeiliSearch is enabled but no configuration is provided'); + } + return new MeiliSearch({ host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`, apiKey: config.meilisearch.apiKey, diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index a5dc61db137f6c9481ff7d74dc95a60865c98bfe..c0b1484804114abb1ddd9f7c3d035aa8cf56a9e3 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -50,6 +50,9 @@ type Source = { redisForJobQueue?: RedisOptionsSource; redisForTimelines?: RedisOptionsSource; redisForReactions?: RedisOptionsSource; + fulltextSearch?: { + provider?: FulltextSearchProvider; + }; meilisearch?: { host: string; port: string; @@ -131,6 +134,9 @@ export type Config = { user: string; pass: string; }[] | undefined; + fulltextSearch?: { + provider?: FulltextSearchProvider; + }; meilisearch: { host: string; port: string; @@ -197,6 +203,8 @@ export type Config = { pidFile: string; }; +export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch'; + const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -265,6 +273,7 @@ export function loadConfig(): Config { db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass }, dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, + fulltextSearch: config.fulltextSearch, meilisearch: config.meilisearch, redis, redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index f3c11277c548acf438b43ae78a586293a416b1bd..64e3f2f56a8e0684deeb8436e15f44b1ab58c072 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,16 +6,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -27,12 +28,24 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + function compileValue(value: V): string { if (typeof value === 'string') { return `'${value}'`; // TODO: escape @@ -64,7 +77,8 @@ function compileQuery(q: Q): string { @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -79,6 +93,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`); @@ -109,132 +124,185 @@ export class SearchService { if (config.meilisearch?.scope) { this.meilisearchIndexScope = config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis public async indexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); - } + await this.meilisearchNoteIndex?.deleteDocument(note.id); } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise<MiNote[]> { - if (this.meilisearch) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': { + // ã»ã¨ã‚“ã©å†…容ã«å·®ãŒãªã„ã®ã§sqlLikeã¨sqlPgroongaã‚’åŒã˜å‡¦ç†ã«ã—ã¦ã„ã‚‹. + // 今後ã®æ‹¡å¼µã§å·®ãŒå‡ºã‚‹ç”¨ã§ã‚ã‚Œã°é–¢æ•°ã‚’分ã‘ã‚‹. + return this.searchNoteByLike(q, me, opts, pagination); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: ['createdAt:desc'], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set<string>(), new Set<string>()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); + } + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } + } + } + + @bindThis + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@ :q', { q }); } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + query.andWhere('LOWER(note.text) LIKE :q', { q: `%${ sqlLikeEscape(q.toLowerCase()) }%` }); + } - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + query.andWhere('user.host IS NULL'); + } else { + query.andWhere('user.host = :host', { host: opts.host }); } + } - query - .andWhere('LOWER(note.text) LIKE :q', { q: `%${ sqlLikeEscape(q.toLowerCase()) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (opts.host) { - if (opts.host === '.') { - query.andWhere('user.host IS NULL'); - } else { - query.andWhere('user.host = :host', { host: opts.host }); - } - } + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + + return query.limit(pagination.limit).getMany(); + } - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); + } + } - return await query.limit(pagination.limit).getMany(); + const res = await this.meilisearchNoteIndex.search(q, { + sort: ['createdAt:desc'], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) + : [new Set<string>(), new Set<string>()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } }