MCPcopy Create free account
hub / github.com/google-research/language / build_encoder_fn

Method build_encoder_fn

language/serene/preprocessing.py:464–540  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

462 }
463
464 def build_encoder_fn(self):
465
466 def encode(claim, evidence, claim_label, evidence_label, metadata,
467 wikipedia_url, sentence_id):
468 """Encode the example using the input TF example.
469
470 Args:
471 claim: Claim text
472 evidence: Evidence text
473 claim_label: Claim verification label
474 evidence_label: Label if is relevant evidence/claim
475 metadata: Metadata about example
476 wikipedia_url: wikipedia_url of example, or None
477 sentence_id: sentence_id of the example, or None
478
479 Returns:
480 Input fields of ClassicEmbedder model
481 """
482 claim_tokens, claim_word_ids, claim_mask, claim_segment_ids = self.encode(
483 claim.numpy())
484 evidence = create_evidence_input(
485 wikipedia_url.numpy().decode('utf8')
486 if self._include_title else None,
487 sentence_id.numpy().decode('utf8')
488 if self._include_sentence_id else None,
489 evidence.numpy().decode('utf8')
490 )
491
492 evidence_tokens, evidence_word_ids, evidence_mask, evidence_segment_ids = self.encode(
493 evidence)
494 return (claim_tokens, claim_word_ids, claim_mask, claim_segment_ids,
495 evidence_tokens, evidence_word_ids, evidence_mask,
496 evidence_segment_ids, claim_label, evidence_label, metadata)
497
498 def encode_map_fn(example):
499 """Read example from tfds and constructed named tensor inputs."""
500 tensors = tf.py_function(
501 encode,
502 inp=[
503 example['claim_text'], example['evidence_text'],
504 example['claim_label'], example['evidence_label'],
505 example['metadata'], example['wikipedia_url'],
506 example['sentence_id']
507 ],
508 Tout=(
509 tf.string,
510 tf.int32,
511 tf.int32,
512 tf.int32,
513 tf.string,
514 tf.int32,
515 tf.int32,
516 tf.int32,
517 tf.int64,
518 tf.int64,
519 tf.string,
520 ))
521 # Keras expects (x, y), where x can be a dictionary.

Callers 1

_encode_datasetMethod · 0.45

Calls

no outgoing calls

Tested by

no test coverage detected