Method load_hf_tokenizer

tensorrt_llm/llmapi/llm_utils.py:550–559

(model_dir,
                          trust_remote_code: bool = True,
                          use_fast: bool = True,
                          **kwargs)

    @staticmethod
    def load_hf_tokenizer(model_dir,
                          trust_remote_code: bool = True,
                          use_fast: bool = True,
                          **kwargs) -> Optional[TransformersTokenizer]:
        if (tokenizer := load_hf_tokenizer(model_dir, trust_remote_code,
                                           use_fast, **kwargs)) is not None:
            return tokenizer
        else:
            logger.warning(f"Failed to load tokenizer from {model_dir}")
            return None

    @staticmethod
    def load_hf_generation_config(