token ids to strings Args: token_ids (List[int]): token ids task_id (str): task id Returns: List[str]: strings
(self, token_id, task_id)
| 612 | return token_ids |
| 613 | |
| 614 | def ids2tokens(self, token_id, task_id): |
| 615 | """ |
| 616 | token ids to strings |
| 617 | |
| 618 | Args: |
| 619 | token_ids (List[int]): token ids |
| 620 | task_id (str): task id |
| 621 | |
| 622 | Returns: |
| 623 | List[str]: strings |
| 624 | """ |
| 625 | if envs.FD_USE_HF_TOKENIZER: |
| 626 | if task_id not in self.decode_status: |
| 627 | # history token ids & history token strings & befer decode str |
| 628 | self.decode_status[task_id] = [[], [], ""] |
| 629 | |
| 630 | previous_token_ids = self.decode_status[task_id][0] |
| 631 | decode_str = self.tokenizer.batch_decode( |
| 632 | [previous_token_ids + token_id], |
| 633 | skip_special_tokens=True, |
| 634 | clean_up_tokenization_spaces=False, |
| 635 | ) |
| 636 | if isinstance(decode_str, list) and len(decode_str): |
| 637 | new_str = decode_str[0].replace(self.decode_status[task_id][2], "", 1) |
| 638 | self.decode_status[task_id][1].append(new_str) |
| 639 | self.decode_status[task_id][2] = decode_str[0] |
| 640 | else: |
| 641 | new_str = "" |
| 642 | self.decode_status[task_id][0] += token_id |
| 643 | return new_str |
| 644 | else: |
| 645 | if task_id not in self.decode_status: |
| 646 | # prefix offset & read offset & history token ids & history token strings |
| 647 | self.decode_status[task_id] = [0, 0, [], ""] |
| 648 | |
| 649 | prefix_offset = self.decode_status[task_id][0] |
| 650 | read_offset = self.decode_status[task_id][1] |
| 651 | previous_token_ids = self.decode_status[task_id][2] |
| 652 | previous_texts = self.decode_status[task_id][3] |
| 653 | decode_str, prefix_offset, read_offset = self.tokenizer.decode_token( |
| 654 | previous_token_ids + token_id, prefix_offset, read_offset |
| 655 | ) |
| 656 | self.decode_status[task_id][0] = prefix_offset |
| 657 | self.decode_status[task_id][1] = read_offset |
| 658 | self.decode_status[task_id][2] += token_id |
| 659 | self.decode_status[task_id][3] += decode_str |
| 660 | |
| 661 | return decode_str, previous_token_ids, previous_texts |
| 662 | |
| 663 | def _load_tokenizer(self): |
| 664 | """ |
no test coverage detected