(self)
| 410 | return self |
| 411 | |
| 412 | def _get_bad_words(self) -> List[List[int]]: |
| 413 | words = [] |
| 414 | if self.bad_token_ids: |
| 415 | words = [[i] for i in self.bad_token_ids] |
| 416 | |
| 417 | if self.bad is None: |
| 418 | return words |
| 419 | else: |
| 420 | if self._bad_word_ids is None: |
| 421 | raise RuntimeError( |
| 422 | f"{self.__class__.__name__}.bad ({self.bad}) is not processed by tokenizer, " |
| 423 | "please call the setup method." |
| 424 | ) |
| 425 | return words + self._bad_word_ids |
| 426 | |
| 427 | def _get_stop_words(self) -> List[List[int]]: |
| 428 | words = [] |