token ids to strings Args: token_ids (List[int]): token ids task_id (str): task id Returns: List[str]: strings
(self, token_id, task_id)
| 623 | return token_ids |
| 624 | |
| 625 | def ids2tokens(self, token_id, task_id): |
| 626 | """ |
| 627 | token ids to strings |
| 628 | |
| 629 | Args: |
| 630 | token_ids (List[int]): token ids |
| 631 | task_id (str): task id |
| 632 | |
| 633 | Returns: |
| 634 | List[str]: strings |
| 635 | """ |
| 636 | if envs.FD_USE_HF_TOKENIZER: |
| 637 | if task_id not in self.decode_status: |
| 638 | # history token ids & history token strings & befer decode str |
| 639 | self.decode_status[task_id] = [[], [], ""] |
| 640 | |
| 641 | previous_token_ids = self.decode_status[task_id][0] |
| 642 | decode_str = self.tokenizer.batch_decode( |
| 643 | [previous_token_ids + token_id], |
| 644 | skip_special_tokens=True, |
| 645 | clean_up_tokenization_spaces=False, |
| 646 | ) |
| 647 | if isinstance(decode_str, list) and len(decode_str): |
| 648 | new_str = decode_str[0].replace(self.decode_status[task_id][2], "", 1) |
| 649 | self.decode_status[task_id][1].append(new_str) |
| 650 | self.decode_status[task_id][2] = decode_str[0] |
| 651 | else: |
| 652 | new_str = "" |
| 653 | self.decode_status[task_id][0] += token_id |
| 654 | return new_str |
| 655 | else: |
| 656 | if task_id not in self.decode_status: |
| 657 | # prefix offset & read offset & history token ids & history token strings |
| 658 | self.decode_status[task_id] = [0, 0, [], ""] |
| 659 | |
| 660 | prefix_offset = self.decode_status[task_id][0] |
| 661 | read_offset = self.decode_status[task_id][1] |
| 662 | previous_token_ids = self.decode_status[task_id][2] |
| 663 | previous_texts = self.decode_status[task_id][3] |
| 664 | decode_str, prefix_offset, read_offset = self.tokenizer.decode_token( |
| 665 | previous_token_ids + token_id, prefix_offset, read_offset |
| 666 | ) |
| 667 | self.decode_status[task_id][0] = prefix_offset |
| 668 | self.decode_status[task_id][1] = read_offset |
| 669 | self.decode_status[task_id][2] += token_id |
| 670 | self.decode_status[task_id][3] += decode_str |
| 671 | |
| 672 | return decode_str, previous_token_ids, previous_texts |
| 673 | |
| 674 | def _load_tokenizer(self): |
| 675 | """ |