(tokenizer, text, add_special_tokens)
| 378 | self.pad_id = self.end_id |
| 379 | |
| 380 | def _encode(tokenizer, text, add_special_tokens): |
| 381 | try: |
| 382 | return tokenizer.encode(text, add_special_tokens=add_special_tokens) |
| 383 | except TypeError: |
| 384 | # For tiktokenizer, the encode method does not have add_special_tokens argument |
| 385 | return tokenizer.encode(text) |
| 386 | |
| 387 | if self.bad is not None: |
| 388 | strs = [self.bad] if isinstance(self.bad, str) else self.bad |