Function fallbackTokenize

packages/node-runtime/src/nlp/fts-tokenizer.ts:28–34

(text: string)

}

function fallbackTokenize(text: string): string {
  return text
    .toLowerCase()
    .split(/\s+/)
    .filter((t) => t.length > 0)
    .join(' ')
}

function escapeToken(token: string): string {
  return `"${token.replace(/"/g, '""')}"`