| 33 | return self.vocab[self.eos] |
| 34 | |
| 35 | def bpe(self, text): |
| 36 | if text in self._cache: |
| 37 | return self._cache[text] |
| 38 | |
| 39 | unigrams = list(text[:-1]) + [text[-1] + "</w>"] |
| 40 | unique_bigrams = set(zip(unigrams, unigrams[1:])) |
| 41 | |
| 42 | if not unique_bigrams: |
| 43 | return unigrams |
| 44 | |
| 45 | # In every iteration try to merge the two most likely bigrams. If none |
| 46 | # was merged we are done. |
| 47 | # |
| 48 | # Ported from https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/tokenization_clip.py |
| 49 | while unique_bigrams: |
| 50 | bigram = min( |
| 51 | unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf")) |
| 52 | ) |
| 53 | if bigram not in self.bpe_ranks: |
| 54 | break |
| 55 | |
| 56 | new_unigrams = [] |
| 57 | skip = False |
| 58 | for a, b in zip(unigrams, unigrams[1:]): |
| 59 | if skip: |
| 60 | skip = False |
| 61 | continue |
| 62 | |
| 63 | if (a, b) == bigram: |
| 64 | new_unigrams.append(a + b) |
| 65 | skip = True |
| 66 | |
| 67 | else: |
| 68 | new_unigrams.append(a) |
| 69 | |
| 70 | if not skip: |
| 71 | new_unigrams.append(b) |
| 72 | |
| 73 | unigrams = new_unigrams |
| 74 | unique_bigrams = set(zip(unigrams, unigrams[1:])) |
| 75 | |
| 76 | self._cache[text] = unigrams |
| 77 | |
| 78 | return unigrams |
| 79 | |
| 80 | def tokenize(self, text, prepend_bos=True, append_eos=True): |
| 81 | if isinstance(text, list): |