(cls, filename_prefix)
| 644 | |
| 645 | @classmethod |
| 646 | def load_from_file(cls, filename_prefix): |
| 647 | with tf.io.gfile.GFile(f'{filename_prefix}.encoder') as f: |
| 648 | params = json.load(f) |
| 649 | bert_tokenizer = tokenizers.BertTokenizer( |
| 650 | vocab_file=params['vocab_file'], |
| 651 | do_lower_case=params['do_lower_case'] |
| 652 | ) |
| 653 | bert_encoder = BertTextEncoder( |
| 654 | tokenizer=bert_tokenizer, |
| 655 | max_seq_length=params['max_seq_length'], |
| 656 | include_title=params['include_title'], |
| 657 | include_sentence_id=params['include_sentence_id'] |
| 658 | ) |
| 659 | return bert_encoder |
| 660 | |
| 661 | @classmethod |
| 662 | def _write_lines_to_file(cls, filename, lines, metadata_dict=None): |
no test coverage detected