diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ccaa9f1e80ed9321c8172be38057d671138902..c4da82e0a1a473afe85a609b276be619eac4fa8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ ### Server - Enhance: タイムラインå–得時ã®ãƒ‘フォーマンスを大幅ã«å‘上 - Enhance: ãƒã‚¤ãƒ©ã‚¤ãƒˆå–得時ã®ãƒ‘フォーマンスを大幅ã«å‘上 +- Enhance: トレンドãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°å–得時ã®ãƒ‘フォーマンスを大幅ã«å‘上 - Enhance: ä¸è¦ãªPostgreSQLã®ã‚¤ãƒ³ãƒ‡ãƒƒã‚¯ã‚¹ã‚’削除ã—パフォーマンスをå‘上 ## 2023.9.3 diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index 945c23b0e24ed371ca59151a28ce021f54be1c48..62b50ed38d0e3de80fe42708a58053742346e832 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3æ—¥ã”㨠const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ã”㨠+const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ã”㨠@Injectable() export class FeaturedService { @@ -88,6 +89,11 @@ export class FeaturedService { return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); } + @bindThis + public updateHashtagsRanking(hashtag: string, score = 1): Promise<void> { + return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score); + } + @bindThis public getGlobalNotesRanking(limit: number): Promise<MiNote['id'][]> { return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, limit); @@ -102,4 +108,9 @@ export class FeaturedService { public getPerUserNotesRanking(userId: MiUser['id'], limit: number): Promise<MiNote['id'][]> { return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, limit); } + + @bindThis + public getHashtagsRanking(limit: number): Promise<string[]> { + return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, limit); + } } diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index c72c7460ff21bb00264e77cb4c37de49e6b7516a..4900fa90a138f898a46d5af08edac8df7a5de1de 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -4,6 +4,8 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { getLineAndCharacterOfPosition } from 'typescript'; import { DI } from '@/di-symbols.js'; import type { MiUser } from '@/models/User.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; @@ -12,14 +14,19 @@ import type { MiHashtag } from '@/models/Hashtag.js'; import type { HashtagsRepository } from '@/models/_.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; @Injectable() export class HashtagService { constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, // TODO: 専用ã®Redisサーãƒãƒ¼ã‚’è¨å®šã§ãるよã†ã«ã™ã‚‹ + @Inject(DI.hashtagsRepository) private hashtagsRepository: HashtagsRepository, private userEntityService: UserEntityService, + private featuredService: FeaturedService, private idService: IdService, ) { } @@ -46,6 +53,9 @@ export class HashtagService { public async updateHashtag(user: { id: MiUser['id']; host: MiUser['host']; }, tag: string, isUserAttached = false, inc = true) { tag = normalizeForSearch(tag); + // TODO: サンプリング + this.updateHashtagsRanking(tag, user.id); + const index = await this.hashtagsRepository.findOneBy({ name: tag }); if (index == null && !inc) return; @@ -85,7 +95,7 @@ export class HashtagService { } } } else { - // 自分ãŒåˆã‚ã¦ã“ã®ã‚¿ã‚°ã‚’使ã£ãŸãªã‚‰ + // 自分ãŒåˆã‚ã¦ã“ã®ã‚¿ã‚°ã‚’使ã£ãŸãªã‚‰ if (!index.mentionedUserIds.some(id => id === user.id)) { set.mentionedUserIds = () => `array_append("mentionedUserIds", '${user.id}')`; set.mentionedUsersCount = () => '"mentionedUsersCount" + 1'; @@ -144,4 +154,93 @@ export class HashtagService { } } } + + @bindThis + public async updateHashtagsRanking(hashtag: string, userId: MiUser['id']): Promise<void> { + // TODO: instance.hiddenTagsã®è€ƒæ…® + + // YYYYMMDDHHmm (10分間隔) + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + + const exist = await this.redisClient.sismember(`hashtagUsers:${hashtag}`, userId); + if (exist === 1) return; + + this.featuredService.updateHashtagsRanking(hashtag, 1); + + const redisPipeline = this.redisClient.pipeline(); + + // TODO: ã“れら㮠Set 㯠Bloom Filter を使ã†ã‚ˆã†ã«ã—ã¦ã‚‚良ã•ãㆠ+ + // ãƒãƒ£ãƒ¼ãƒˆç”¨ + redisPipeline.sadd(`hashtagUsers:${hashtag}:${window}`, userId); + redisPipeline.expire(`hashtagUsers:${hashtag}:${window}`, + 60 * 60 * 24 * 3, // 3日間 + 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期é™ãŒãªã„ã¨ãã ã‘è¨å®š + ); + + // ユニークカウント用 + redisPipeline.sadd(`hashtagUsers:${hashtag}`, userId); + redisPipeline.expire(`hashtagUsers:${hashtag}`, + 60 * 60, // 1時間 + 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期é™ãŒãªã„ã¨ãã ã‘è¨å®š + ); + + redisPipeline.exec(); + } + + @bindThis + public async getChart(hashtag: string, range: number): Promise<number[]> { + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + + const redisPipeline = this.redisClient.pipeline(); + + for (let i = 0; i < range; i++) { + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`); + now.setMinutes(now.getMinutes() - (i * 10), 0, 0); + } + + const result = await redisPipeline.exec(); + + if (result == null) return []; + + return result.map(x => x[1]) as number[]; + } + + @bindThis + public async getCharts(hashtags: string[], range: number): Promise<Record<string, number[]>> { + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + + const redisPipeline = this.redisClient.pipeline(); + + for (let i = 0; i < range; i++) { + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + for (const hashtag of hashtags) { + redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`); + } + now.setMinutes(now.getMinutes() - (i * 10), 0, 0); + } + + const result = await redisPipeline.exec(); + + if (result == null) return {}; + + // key is hashtag + const charts = {} as Record<string, number[]>; + for (const hashtag of hashtags) { + charts[hashtag] = []; + } + + for (let i = 0; i < range; i++) { + for (let j = 0; j < hashtags.length; j++) { + charts[hashtags[j]].push(result[(i * hashtags.length) + j][1] as number); + } + } + + return charts; + } } diff --git a/packages/backend/src/server/api/endpoints/hashtags/trend.ts b/packages/backend/src/server/api/endpoints/hashtags/trend.ts index 75d4fe3819b387de8b5c4719eef74c907c6ab2c3..a69e007a400569482f17b923ee0517d0fbc440d5 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/trend.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/trend.ts @@ -3,29 +3,13 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { NotesRepository } from '@/models/_.js'; -import type { MiNote } from '@/models/Note.js'; -import { safeForSql } from '@/misc/safe-for-sql.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { MetaService } from '@/core/MetaService.js'; import { DI } from '@/di-symbols.js'; - -/* -トレンドã«è¼‰ã‚‹ãŸã‚ã«ã¯ã€Œã€Žç›´è¿‘a分間ã®ãƒ¦ãƒ‹ãƒ¼ã‚¯æŠ•ç¨¿æ•°ãŒä»Šã‹ã‚‰a分å‰ï½žä»Šã‹ã‚‰b分å‰ã®é–“ã®ãƒ¦ãƒ‹ãƒ¼ã‚¯æŠ•ç¨¿æ•°ã®nå€ä»¥ä¸Šã€ã®ãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°ã®ä¸Šä½5ä½ä»¥å†…ã«å…¥ã‚‹ã€ã“ã¨ãŒå¿…è¦ -ユニーク投稿数ã¨ã¯ãã®ãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°ã¨æŠ•ç¨¿ãƒ¦ãƒ¼ã‚¶ãƒ¼ã®ãƒšã‚¢ã®ã‚«ã‚¦ãƒ³ãƒˆã§ã€ä¾‹ãˆã°åŒã˜ãƒ¦ãƒ¼ã‚¶ãƒ¼ãŒè¤‡æ•°å›žåŒã˜ãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°ã‚’投稿ã—ã¦ã‚‚ãã®ãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°ã®ãƒ¦ãƒ‹ãƒ¼ã‚¯æŠ•ç¨¿æ•°ã¯1ã¨ã‚«ã‚¦ãƒ³ãƒˆã•ã‚Œã‚‹ - -..ãŒç†æƒ³ã ã‘ã©PostgreSQLã§ã©ã†ã™ã‚‹ã®ã‹åˆ†ã‹ã‚‰ãªã„ã®ã§å˜ã«ã€Œç›´è¿‘Aã®å†…ã«æŠ•ç¨¿ã•ã‚ŒãŸãƒ¦ãƒ‹ãƒ¼ã‚¯æŠ•ç¨¿æ•°ãŒå¤šã„ãƒãƒƒã‚·ãƒ¥ã‚¿ã‚°ã€ã§å¦¥å”ã™ã‚‹ -*/ - -const rangeA = 1000 * 60 * 60; // 60分 -//const rangeB = 1000 * 60 * 120; // 2時間 -//const coefficient = 1.25; // 「nå€ã€ã®éƒ¨åˆ† -//const requiredUsers = 3; // 最低何人ãŒãã®ã‚¿ã‚°ã‚’投稿ã—ã¦ã„ã‚‹å¿…è¦ãŒã‚ã‚‹ã‹ - -const max = 5; +import { FeaturedService } from '@/core/FeaturedService.js'; +import { HashtagService } from '@/core/HashtagService.js'; export const meta = { tags: ['hashtags'], @@ -71,98 +55,22 @@ export const paramDef = { @Injectable() export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.notesRepository) - private notesRepository: NotesRepository, - private metaService: MetaService, + private featuredService: FeaturedService, + private hashtagService: HashtagService, ) { super(meta, paramDef, async () => { const instance = await this.metaService.fetch(true); const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); - const now = new Date(); // 5分å˜ä½ã§ä¸¸ã‚ãŸç¾åœ¨æ—¥æ™‚ - now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0); - - const tagNotes = await this.notesRepository.createQueryBuilder('note') - .where('note.createdAt > :date', { date: new Date(now.getTime() - rangeA) }) - .andWhere(new Brackets(qb => { qb - .where('note.visibility = \'public\'') - .orWhere('note.visibility = \'home\''); - })) - .andWhere('note.tags != \'{}\'') - .select(['note.tags', 'note.userId']) - .cache(60000) // 1 min - .getMany(); - - if (tagNotes.length === 0) { - return []; - } - - const tags: { - name: string; - users: MiNote['userId'][]; - }[] = []; - - for (const note of tagNotes) { - for (const tag of note.tags) { - if (hiddenTags.includes(tag)) continue; - - const x = tags.find(x => x.name === tag); - if (x) { - if (!x.users.includes(note.userId)) { - x.users.push(note.userId); - } - } else { - tags.push({ - name: tag, - users: [note.userId], - }); - } - } - } - - // ã‚¿ã‚°ã‚’äººæ°—é †ã«ä¸¦ã¹æ›¿ãˆ - const hots = tags - .sort((a, b) => b.users.length - a.users.length) - .map(tag => tag.name) - .slice(0, max); - - //#region 2(ã¾ãŸã¯3)ã§è©±é¡Œã¨åˆ¤å®šã•ã‚ŒãŸã‚¿ã‚°ãã‚Œãžã‚Œã«ã¤ã„ã¦éŽåŽ»ã®æŠ•ç¨¿æ•°ã‚°ãƒ©ãƒ•ã‚’å–å¾—ã™ã‚‹ - const countPromises: Promise<number[]>[] = []; - - const range = 20; - - // 10分 - const interval = 1000 * 60 * 10; - - for (let i = 0; i < range; i++) { - countPromises.push(Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note') - .select('count(distinct note.userId)') - .where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`) - .andWhere('note.createdAt < :lt', { lt: new Date(now.getTime() - (interval * i)) }) - .andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - (interval * (i + 1))) }) - .cache(60000) // 1 min - .getRawOne() - .then(x => parseInt(x.count, 10)), - ))); - } - - const countsLog = await Promise.all(countPromises); - //#endregion + const ranking = await this.featuredService.getHashtagsRanking(10); - const totalCounts = await Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note') - .select('count(distinct note.userId)') - .where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`) - .andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - rangeA) }) - .cache(60000 * 60) // 60 min - .getRawOne() - .then(x => parseInt(x.count, 10)), - )); + const charts = ranking.length === 0 ? {} : await this.hashtagService.getCharts(ranking, 20); - const stats = hots.map((tag, i) => ({ + const stats = ranking.map((tag, i) => ({ tag, - chart: countsLog.map(counts => counts[i]), - usersCount: totalCounts[i], + chart: charts[tag], + usersCount: Math.max(...charts[tag]), })); return stats;