| 140 | } |
| 141 | |
| 142 | std::shared_ptr<Graph<int64_t>> Tokenizer::tokenize( |
| 143 | const std::string& input) const { |
| 144 | return ::mlx::data::core::tokenize(trie_, input, ignoreUnk_); |
| 145 | } |
| 146 | |
| 147 | std::vector<int64_t> Tokenizer::tokenize_shortest( |
| 148 | const std::string& input) const { |