(db: DatabaseAdapter, params: WordFrequencyParams)
| 51 | * 从数据库计算词频统计 |
| 52 | */ |
| 53 | export function computeWordFrequency(db: DatabaseAdapter, params: WordFrequencyParams): WordFrequencyResult { |
| 54 | const { |
| 55 | locale, |
| 56 | timeFilter, |
| 57 | memberId, |
| 58 | topN = 100, |
| 59 | minWordLength, |
| 60 | minCount = 2, |
| 61 | posFilterMode = 'meaningful', |
| 62 | customPosTags, |
| 63 | enableStopwords = true, |
| 64 | dictType = 'default', |
| 65 | excludeWords, |
| 66 | } = params |
| 67 | |
| 68 | const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId) |
| 69 | |
| 70 | const messages = db |
| 71 | .prepare(`SELECT msg.content FROM message msg JOIN member m ON msg.sender_id = m.id${clause}`) |
| 72 | .all(...filterParams) as Array<{ content: string }> |
| 73 | |
| 74 | if (messages.length === 0) { |
| 75 | return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 } |
| 76 | } |
| 77 | |
| 78 | const texts = messages.map((m) => m.content) |
| 79 | |
| 80 | const segmentOptions = { |
| 81 | minLength: minWordLength, |
| 82 | minCount, |
| 83 | topN, |
| 84 | posFilterMode, |
| 85 | customPosTags, |
| 86 | enableStopwords, |
| 87 | dictType: dictType as DictType, |
| 88 | excludeWords, |
| 89 | } |
| 90 | |
| 91 | let posTagStats: PosTagStat[] | undefined |
| 92 | let result: BatchSegmentResult |
| 93 | |
| 94 | // 中文 meaningful/custom 模式下，词频与词性统计可在一次分词内同时产出，避免重复分词。 |
| 95 | if (locale.startsWith('zh') && posFilterMode !== 'all') { |
| 96 | const combined = batchSegmentChineseWithStats(texts, locale as SupportedLocale, segmentOptions) |
| 97 | result = { words: combined.words, uniqueWords: combined.uniqueWords, totalWords: combined.totalWords } |
| 98 | posTagStats = [...combined.posTagStats.entries()].map(([tag, count]) => ({ tag, count })) |
| 99 | } else { |
| 100 | if (locale.startsWith('zh')) { |
| 101 | const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType as DictType) |
| 102 | posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count })) |
| 103 | } |
| 104 | result = batchSegmentWithFrequency(texts, locale as SupportedLocale, segmentOptions) |
| 105 | } |
| 106 | |
| 107 | let topNTotalWords = 0 |
| 108 | for (const count of result.words.values()) topNTotalWords += count |
| 109 | |
| 110 | const words = [...result.words.entries()].map(([word, count]) => ({ |
no test coverage detected