MCPcopy Create free account
hub / github.com/google-research/language / BertTextEncoder

Class BertTextEncoder

language/serene/preprocessing.py:423–706  ·  view source on GitHub ↗

Bert Text encoder.

Source from the content-addressed store, hash-verified

421
422
423class BertTextEncoder(FeverTextEncoder):
424 """Bert Text encoder."""
425
426 def __init__(
427 self, *,
428 tokenizer, max_seq_length,
429 include_title, include_sentence_id):
430 super().__init__()
431 self._tokenizer = tokenizer
432 self._max_seq_length = max_seq_length
433 self._include_title = include_title
434 self._include_sentence_id = include_sentence_id
435 self._cls_id = self._tokenizer.vocab['[CLS]']
436 self._sep_id = self._tokenizer.vocab['[SEP]']
437 self._pad_id = self._tokenizer.vocab['[PAD]']
438
439 def encode_example(self, *, claim, evidence, claim_label, evidence_label,
440 metadata, wikipedia_url, sentence_id):
441 claim_tokens, claim_word_ids, claim_mask, claim_segment_ids = self.encode(
442 claim)
443 evidence = create_evidence_input(
444 wikipedia_url if self._include_title else None,
445 sentence_id if self._include_sentence_id else None,
446 evidence
447 )
448 evidence_tokens, evidence_word_ids, evidence_mask, evidence_segment_ids = self.encode(
449 evidence)
450 return {
451 'claim_text_tokens': claim_tokens,
452 'claim_text_word_ids': claim_word_ids,
453 'claim_text_mask': claim_mask,
454 'claim_text_segment_ids': claim_segment_ids,
455 'evidence_text_tokens': evidence_tokens,
456 'evidence_text_word_ids': evidence_word_ids,
457 'evidence_text_mask': evidence_mask,
458 'evidence_text_segment_ids': evidence_segment_ids,
459 'claim_label': claim_label,
460 'evidence_label': evidence_label,
461 'metadata': metadata
462 }
463
464 def build_encoder_fn(self):
465
466 def encode(claim, evidence, claim_label, evidence_label, metadata,
467 wikipedia_url, sentence_id):
468 """Encode the example using the input TF example.
469
470 Args:
471 claim: Claim text
472 evidence: Evidence text
473 claim_label: Claim verification label
474 evidence_label: Label if is relevant evidence/claim
475 metadata: Metadata about example
476 wikipedia_url: wikipedia_url of example, or None
477 sentence_id: sentence_id of the example, or None
478
479 Returns:
480 Input fields of ClassicEmbedder model

Callers 1

load_from_fileMethod · 0.85

Calls

no outgoing calls

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…