(self)
| 38 | test_rust_tokenizer = True |
| 39 | |
| 40 | def setUp(self): |
| 41 | super().setUp() |
| 42 | |
| 43 | vocab_tokens = [ |
| 44 | "[UNK]", |
| 45 | "[CLS]", |
| 46 | "[SEP]", |
| 47 | "[PAD]", |
| 48 | "[MASK]", |
| 49 | "want", |
| 50 | "##want", |
| 51 | "##ed", |
| 52 | "wa", |
| 53 | "un", |
| 54 | "runn", |
| 55 | "##ing", |
| 56 | ",", |
| 57 | "low", |
| 58 | "lowest", |
| 59 | ] |
| 60 | self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) |
| 61 | with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: |
| 62 | vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) |
| 63 | |
| 64 | def get_rust_tokenizer(self, **kwargs): |
| 65 | return BertTokenizerFast.from_pretrained(self.tmpdirname, **kwargs) |
nothing calls this directly
no outgoing calls
no test coverage detected