Constructs a TokenTextEncoder. To load from a file saved with `TokenTextEncoder.save_to_file`, use `TokenTextEncoder.load_from_file`. Args: vocab_list: `list `, list of tokens. tokenizer: `Tokenizer`, responsible for converting incoming text into a list of token
(
self,
*,
vocab_list,
tokenizer,
include_title,
include_sentence_id,
oov_buckets = 1,
oov_token = 'UNK',
lowercase = False,
strip_vocab = True,
decode_token_separator = ' ',
max_claim_tokens = None,
max_evidence_tokens = None)
| 184 | """ |
| 185 | |
| 186 | def __init__( |
| 187 | self, |
| 188 | *, |
| 189 | vocab_list, |
| 190 | tokenizer, |
| 191 | include_title, |
| 192 | include_sentence_id, |
| 193 | oov_buckets = 1, |
| 194 | oov_token = 'UNK', |
| 195 | lowercase = False, |
| 196 | strip_vocab = True, |
| 197 | decode_token_separator = ' ', |
| 198 | max_claim_tokens = None, |
| 199 | max_evidence_tokens = None): |
| 200 | """Constructs a TokenTextEncoder. |
| 201 | |
| 202 | To load from a file saved with `TokenTextEncoder.save_to_file`, use |
| 203 | `TokenTextEncoder.load_from_file`. |
| 204 | |
| 205 | Args: |
| 206 | vocab_list: `list<str>`, list of tokens. |
| 207 | tokenizer: `Tokenizer`, responsible for converting incoming text into a |
| 208 | list of tokens. |
| 209 | include_title: Whether to include wikipedia title in evidence |
| 210 | include_sentence_id: Whether to include sentence_id in evidence |
| 211 | oov_buckets: `int`, the number of `int`s to reserve for OOV hash buckets. |
| 212 | Tokens that are OOV will be hash-modded into a OOV bucket in `encode`. |
| 213 | oov_token: `str`, the string to use for OOV ids in `decode`. |
| 214 | lowercase: `bool`, whether to make all text and tokens lowercase. |
| 215 | strip_vocab: `bool`, whether to strip whitespace from the beginning and |
| 216 | end of elements of `vocab_list`. |
| 217 | decode_token_separator: `str`, the string used to separate tokens when |
| 218 | decoding. |
| 219 | max_claim_tokens: maximum number of claim tokens to use |
| 220 | max_evidence_tokens: maximum number of evidence tokens to use |
| 221 | """ |
| 222 | self._max_claim_tokens = max_claim_tokens |
| 223 | self._max_evidence_tokens = max_evidence_tokens |
| 224 | self._include_title = include_title |
| 225 | self._include_sentence_id = include_sentence_id |
| 226 | self._vocab_list = [tf.compat.as_text(el) for el in vocab_list] |
| 227 | if strip_vocab: |
| 228 | self._vocab_list = [el.strip() for el in self._vocab_list] |
| 229 | self._lowercase = lowercase |
| 230 | if self._lowercase: |
| 231 | self._vocab_list = [t.lower() for t in self._vocab_list] |
| 232 | # Note that internally everything is 0-indexed. Padding is dealt with at the |
| 233 | # end of encode and the beginning of decode. |
| 234 | self._token_to_id = dict( |
| 235 | zip(self._vocab_list, range(len(self._vocab_list)))) |
| 236 | self._oov_buckets = oov_buckets |
| 237 | self._oov_token = tf.compat.as_text(oov_token) |
| 238 | |
| 239 | self._tokenizer = tokenizer |
| 240 | self._decode_token_separator = decode_token_separator |
| 241 | |
| 242 | def encode(self, s): |
| 243 | s = tf.compat.as_text(s) |
nothing calls this directly
no outgoing calls
no test coverage detected