(self)
| 41 | **kwargs) |
| 42 | |
| 43 | def _build_model(self): |
| 44 | BaseLLM._build_model(self) |
| 45 | assert self._engine_dir is None |
| 46 | |
| 47 | # Tokenizer loading should be after calling model_loader(), since model_loader() may download the model from HF hub. |
| 48 | # It should also be before bindings ExecutorConfig, which may depend on tokenizer info. |
| 49 | self._tokenizer = self._try_load_tokenizer() |
| 50 | |
| 51 | # Multimodal special handling: |
| 52 | # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor |
| 53 | # 2. May need to modify model weights for MM (e.g., resize vocab embedding). We must do such operation via input processor's __init__ |
| 54 | checkpoint_format = getattr(self.args, "checkpoint_format", None) |
| 55 | self.input_processor = create_input_processor(self._hf_model_dir, |
| 56 | self.tokenizer, |
| 57 | checkpoint_format) |
| 58 | self._tokenizer = self.input_processor.tokenizer |
| 59 | |
| 60 | assert isinstance(self.args, TorchLlmArgs) |
| 61 | self.args.mm_encoder_only = True |
| 62 | |
| 63 | self._executor = self._executor_cls.create( |
| 64 | self._engine_dir, |
| 65 | executor_config=None, |
| 66 | model_world_size=self.args.parallel_config.world_size, |
| 67 | mpi_session=self.mpi_session, |
| 68 | reuse_mpi_comm=external_mpi_comm_available( |
| 69 | self.args.parallel_config.world_size), |
| 70 | is_llm_executor=True, # TODO: check if this is correct or needed |
| 71 | hf_model_dir=self._hf_model_dir, |
| 72 | tokenizer=self.tokenizer, |
| 73 | llm_args=self.args) |
| 74 | |
| 75 | def _validate_mm_args_for_torch_backend(self, kwargs: dict) -> None: |
| 76 | """Validate that users don't pass LLM-specific arguments when using MultimodalEncoder (PyTorch). |
nothing calls this directly
no test coverage detected