Initialize a tokenizer. Args: model_name (str): The name of the HuggingFace model to pull a tokenizer from. Returns: PreTrainedTokenizer: An initialized HuggingFace tokenizer.
(model_name: str)
| 12 | |
| 13 | |
| 14 | def initialize_tokenizer(model_name: str) -> PreTrainedTokenizer: |
| 15 | """Initialize a tokenizer. |
| 16 | |
| 17 | Args: |
| 18 | model_name (str): The name of the HuggingFace model to pull a |
| 19 | tokenizer from. |
| 20 | |
| 21 | Returns: |
| 22 | PreTrainedTokenizer: An initialized HuggingFace tokenizer. |
| 23 | """ |
| 24 | # Initialize the tokenizer specific to the model that we are planning |
| 25 | # to benchmark. |
| 26 | tokenizer = AutoTokenizer.from_pretrained(model_name, |
| 27 | padding_side="left", |
| 28 | trust_remote_code=True) |
| 29 | if tokenizer.pad_token_id is None: |
| 30 | tokenizer.add_special_tokens({"pad_token": "[PAD]"}) |
| 31 | |
| 32 | return tokenizer |
| 33 | |
| 34 | |
| 35 | def create_dataset_from_stream( |
no test coverage detected