()
| 237 | tokens = [token.strip() for token in f.readlines()] |
| 238 | |
| 239 | def token_gen(): |
| 240 | for token in tokens: |
| 241 | yield token |
| 242 | |
| 243 | self._init_vocab(token_gen(), add_reserved_tokens=False) |
| 244 |
nothing calls this directly
no outgoing calls
no test coverage detected