(self, ids)
| 258 | return tfds.deprecated.text.text_encoder.pad_incr(ids) |
| 259 | |
| 260 | def decode(self, ids): |
| 261 | # This decrements the ids of all words in ids by one, to ensure that 0 |
| 262 | # is preserved as the padding token |
| 263 | # *Also*, this strips padding tokens |
| 264 | ids = tfds.deprecated.text.text_encoder.pad_decr(ids) |
| 265 | |
| 266 | tokens = [] |
| 267 | for int_id in ids: |
| 268 | if int_id < len(self._vocab_list): |
| 269 | tokens.append(self._vocab_list[int_id]) |
| 270 | else: |
| 271 | tokens.append(self._oov_token) |
| 272 | return self._decode_token_separator.join(tokens) |
| 273 | |
| 274 | @property |
| 275 | def vocab_size(self): |
nothing calls this directly
no outgoing calls
no test coverage detected