Function tokenizeForFts

packages/node-runtime/src/nlp/fts-tokenizer.ts:13–26

(text: string | null | undefined)

import { getJieba } from './segmenter'

export function tokenizeForFts(text: string | null | undefined): string {
  if (!text || text.trim().length === 0) return ''

  try {
    const jieba = getJieba()
    const tokens = jieba.cut(text, false)
    return tokens
      .map((t) => t.trim().toLowerCase())
      .filter((t) => t.length > 0)
      .join(' ')
  } catch {
    return fallbackTokenize(text)
  }
}

function fallbackTokenize(text: string): string {
  return text