Converts a space-separated string of tokens to a list of ids.
(self, s)
| 195 | self.seg_index = self._token_to_id[SEG] if SEG in self._token_to_id else self.eos_index |
| 196 | |
| 197 | def encode(self, s): |
| 198 | """Converts a space-separated string of tokens to a list of ids.""" |
| 199 | sentence = s |
| 200 | tokens = sentence.strip().split() |
| 201 | if self._replace_oov is not None: |
| 202 | tokens = [t if t in self._token_to_id else self._replace_oov |
| 203 | for t in tokens] |
| 204 | ret = [self._token_to_id[tok] for tok in tokens] |
| 205 | return ret[::-1] if self._reverse else ret |
| 206 | |
| 207 | def decode(self, ids, strip_eos=False, strip_padding=False): |
| 208 | if strip_padding and self.pad() in list(ids): |
nothing calls this directly
no outgoing calls
no test coverage detected