Encodes each byte to an id. For 8-bit strings only.
| 103 | |
| 104 | |
| 105 | class ByteTextEncoder(TextEncoder): |
| 106 | """Encodes each byte to an id. For 8-bit strings only.""" |
| 107 | |
| 108 | def encode(self, s): |
| 109 | numres = self._num_reserved_ids |
| 110 | if six.PY2: |
| 111 | if isinstance(s, unicode): |
| 112 | s = s.encode("utf-8") |
| 113 | return [ord(c) + numres for c in s] |
| 114 | # Python3: explicitly convert to UTF-8 |
| 115 | return [c + numres for c in s.encode("utf-8")] |
| 116 | |
| 117 | def decode(self, ids, strip_extraneous=False): |
| 118 | if strip_extraneous: |
| 119 | ids = strip_ids(ids, list(range(self._num_reserved_ids or 0))) |
| 120 | numres = self._num_reserved_ids |
| 121 | decoded_ids = [] |
| 122 | int2byte = six.int2byte |
| 123 | for id_ in ids: |
| 124 | if 0 <= id_ < numres: |
| 125 | decoded_ids.append(RESERVED_TOKENS_BYTES[int(id_)]) |
| 126 | else: |
| 127 | decoded_ids.append(int2byte(id_ - numres)) |
| 128 | if six.PY2: |
| 129 | return "".join(decoded_ids) |
| 130 | # Python3: join byte arrays and then decode string |
| 131 | return b"".join(decoded_ids).decode("utf-8", "replace") |
| 132 | |
| 133 | def decode_list(self, ids): |
| 134 | numres = self._num_reserved_ids |
| 135 | decoded_ids = [] |
| 136 | int2byte = six.int2byte |
| 137 | for id_ in ids: |
| 138 | if 0 <= id_ < numres: |
| 139 | decoded_ids.append(RESERVED_TOKENS_BYTES[int(id_)]) |
| 140 | else: |
| 141 | decoded_ids.append(int2byte(id_ - numres)) |
| 142 | # Python3: join byte arrays and then decode string |
| 143 | return decoded_ids |
| 144 | |
| 145 | @property |
| 146 | def vocab_size(self): |
| 147 | return 2**8 + self._num_reserved_ids |
| 148 | |
| 149 | |
| 150 | class ByteTextEncoderWithEos(ByteTextEncoder): |
nothing calls this directly
no outgoing calls
no test coverage detected