Function tokenize

backend/src/utils/text.ts:37–44

(text: string)

const tok_pat = /[a-z0-9]+/gi;

export const tokenize = (text: string): string[] => {
    const toks: string[] = [];
    let m: RegExpExecArray | null;
    while ((m = tok_pat.exec(text))) {
        toks.push(m[0].toLowerCase());
    }
    return toks;
};

const stem = (tok: string): string => {
    if (tok.length <= 3) return tok;