MCPcopy Create free account
hub / github.com/google-research/language / __init__

Method __init__

language/serene/preprocessing.py:186–240  ·  view source on GitHub ↗

Constructs a TokenTextEncoder. To load from a file saved with `TokenTextEncoder.save_to_file`, use `TokenTextEncoder.load_from_file`. Args: vocab_list: `list `, list of tokens. tokenizer: `Tokenizer`, responsible for converting incoming text into a list of token

(
      self,
      *,
      vocab_list,
      tokenizer,
      include_title,
      include_sentence_id,
      oov_buckets = 1,
      oov_token = 'UNK',
      lowercase = False,
      strip_vocab = True,
      decode_token_separator = ' ',
      max_claim_tokens = None,
      max_evidence_tokens = None)

Source from the content-addressed store, hash-verified

184 """
185
186 def __init__(
187 self,
188 *,
189 vocab_list,
190 tokenizer,
191 include_title,
192 include_sentence_id,
193 oov_buckets = 1,
194 oov_token = 'UNK',
195 lowercase = False,
196 strip_vocab = True,
197 decode_token_separator = ' ',
198 max_claim_tokens = None,
199 max_evidence_tokens = None):
200 """Constructs a TokenTextEncoder.
201
202 To load from a file saved with `TokenTextEncoder.save_to_file`, use
203 `TokenTextEncoder.load_from_file`.
204
205 Args:
206 vocab_list: `list<str>`, list of tokens.
207 tokenizer: `Tokenizer`, responsible for converting incoming text into a
208 list of tokens.
209 include_title: Whether to include wikipedia title in evidence
210 include_sentence_id: Whether to include sentence_id in evidence
211 oov_buckets: `int`, the number of `int`s to reserve for OOV hash buckets.
212 Tokens that are OOV will be hash-modded into a OOV bucket in `encode`.
213 oov_token: `str`, the string to use for OOV ids in `decode`.
214 lowercase: `bool`, whether to make all text and tokens lowercase.
215 strip_vocab: `bool`, whether to strip whitespace from the beginning and
216 end of elements of `vocab_list`.
217 decode_token_separator: `str`, the string used to separate tokens when
218 decoding.
219 max_claim_tokens: maximum number of claim tokens to use
220 max_evidence_tokens: maximum number of evidence tokens to use
221 """
222 self._max_claim_tokens = max_claim_tokens
223 self._max_evidence_tokens = max_evidence_tokens
224 self._include_title = include_title
225 self._include_sentence_id = include_sentence_id
226 self._vocab_list = [tf.compat.as_text(el) for el in vocab_list]
227 if strip_vocab:
228 self._vocab_list = [el.strip() for el in self._vocab_list]
229 self._lowercase = lowercase
230 if self._lowercase:
231 self._vocab_list = [t.lower() for t in self._vocab_list]
232 # Note that internally everything is 0-indexed. Padding is dealt with at the
233 # end of encode and the beginning of decode.
234 self._token_to_id = dict(
235 zip(self._vocab_list, range(len(self._vocab_list))))
236 self._oov_buckets = oov_buckets
237 self._oov_token = tf.compat.as_text(oov_token)
238
239 self._tokenizer = tokenizer
240 self._decode_token_separator = decode_token_separator
241
242 def encode(self, s):
243 s = tf.compat.as_text(s)

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected