format of word_dict len(word_dict) should be same to batch_size word_dict[i] means the words for batch i len(word_dict[i]) >= 1, which means it must contain at least 1 string For example, word_dict[2] = [" I am happy", " I am sad"].
(word_dict: List[List[str]],
tokenizer=None,
add_special_tokens=False)
| 60 | |
| 61 | |
| 62 | def decode_words_list(word_dict: List[List[str]], |
| 63 | tokenizer=None, |
| 64 | add_special_tokens=False): |
| 65 | ''' |
| 66 | format of word_dict |
| 67 | len(word_dict) should be same to batch_size |
| 68 | word_dict[i] means the words for batch i |
| 69 | len(word_dict[i]) >= 1, which means it must contain at least 1 string |
| 70 | For example, word_dict[2] = [" I am happy", " I am sad"]. |
| 71 | ''' |
| 72 | assert tokenizer != None, "need to set tokenizer" |
| 73 | |
| 74 | decoded_words_batch = [] |
| 75 | for word_dict_item in word_dict: |
| 76 | decoded_words_request = [] |
| 77 | |
| 78 | for item in word_dict_item: |
| 79 | if isinstance(item, bytes): |
| 80 | item = [item.decode()] |
| 81 | |
| 82 | ids = tokenizer.encode(item, add_special_tokens=add_special_tokens) |
| 83 | |
| 84 | if len(ids) == 0: |
| 85 | continue |
| 86 | |
| 87 | decoded_words_request.append(ids) |
| 88 | decoded_words_batch.append(decoded_words_request) |
| 89 | |
| 90 | return decoded_words_batch |
| 91 | |
| 92 | |
| 93 | def to_word_list_format(word_dict: List[List[List[int]]]): |