Function load_hf_tokenizer

tensorrt_llm/tokenizer/tokenizer.py:416–450

Load a tokenizer from a Hugging Face model directory. Args: model_dir (str): The model directory. trust_remote_code (bool): Whether to trust the remote code. use_fast (bool): Whether to use the fast tokenizer. Returns: A TransformersTokenizer object if the

(model_dir: str,
                      trust_remote_code: bool = True,
                      use_fast: bool = True,
                      **kwargs)

def load_hf_tokenizer(model_dir: str,
                      trust_remote_code: bool = True,
                      use_fast: bool = True,
                      **kwargs) -> Optional[TransformersTokenizer]:
    ''' Load a tokenizer from a Hugging Face model directory.

    Args:
        model_dir (str): The model directory.
        trust_remote_code (bool): Whether to trust the remote code.
        use_fast (bool): Whether to use the fast tokenizer.

    Returns:
        A TransformersTokenizer object if the tokenizer is loaded successfully.
    '''

    try:
        tokenizer = TransformersTokenizer.from_pretrained(
            model_dir,
            legacy=False,
            padding_side='left',
            truncation_side='left',
            trust_remote_code=trust_remote_code,
            use_fast=use_fast,
            **kwargs)

        if trust_remote_code:
            maybe_register_transformers_modules_by_value()

        return tokenizer

    except Exception as e:
        logger.warning(
            f"Failed to load hf tokenizer from {model_dir}, encounter error: {e}"
        )
        return None