| 78 | return unigrams |
| 79 | |
| 80 | def tokenize(self, text, prepend_bos=True, append_eos=True): |
| 81 | if isinstance(text, list): |
| 82 | return [self.tokenize(t, prepend_bos, append_eos) for t in text] |
| 83 | |
| 84 | # Lower case cleanup and split according to self.pat. Hugging Face does |
| 85 | # a much more thorough job here but this should suffice for 95% of |
| 86 | # cases. |
| 87 | clean_text = regex.sub(r"\s+", " ", text.lower()) |
| 88 | tokens = regex.findall(self.pat, clean_text) |
| 89 | |
| 90 | # Split the tokens according to the byte-pair merge file |
| 91 | bpe_tokens = [ti for t in tokens for ti in self.bpe(t)] |
| 92 | |
| 93 | # Map to token ids and return |
| 94 | tokens = [self.vocab[t] for t in bpe_tokens] |
| 95 | if prepend_bos: |
| 96 | tokens = [self.bos_token] + tokens |
| 97 | if append_eos: |
| 98 | tokens.append(self.eos_token) |
| 99 | |
| 100 | return tokens |