(
self,
model: str,
revision: Optional[str] = "master",
tokenizer: Optional[str] = None,
enable_logprob: Optional[bool] = False,
chat_template: Optional[str] = None,
**kwargs,
)
| 80 | """ |
| 81 | |
| 82 | def __init__( |
| 83 | self, |
| 84 | model: str, |
| 85 | revision: Optional[str] = "master", |
| 86 | tokenizer: Optional[str] = None, |
| 87 | enable_logprob: Optional[bool] = False, |
| 88 | chat_template: Optional[str] = None, |
| 89 | **kwargs, |
| 90 | ): |
| 91 | deprecated_kwargs_warning(**kwargs) |
| 92 | |
| 93 | model = retrive_model_from_server(model, revision) |
| 94 | tool_parser_plugin = kwargs.get("tool_parser_plugin") |
| 95 | if tool_parser_plugin: |
| 96 | ToolParserManager.import_tool_parser(tool_parser_plugin) |
| 97 | engine_args = EngineArgs( |
| 98 | model=model, |
| 99 | tokenizer=tokenizer, |
| 100 | enable_logprob=enable_logprob, |
| 101 | **kwargs, |
| 102 | ) |
| 103 | |
| 104 | # Create the Engine |
| 105 | self.llm_engine = LLMEngine.from_engine_args(engine_args=engine_args) |
| 106 | |
| 107 | self.default_sampling_params = SamplingParams(max_tokens=self.llm_engine.cfg.model_config.max_model_len) |
| 108 | |
| 109 | self.llm_engine.start() |
| 110 | |
| 111 | self.mutex = threading.Lock() |
| 112 | self.req_output = dict() |
| 113 | self.master_node_ip = self.llm_engine.cfg.master_ip |
| 114 | self._receive_output_thread = threading.Thread(target=self._receive_output, daemon=True) |
| 115 | self._receive_output_thread.start() |
| 116 | self.chat_template = load_chat_template(chat_template, model) |
| 117 | |
| 118 | def _check_master(self): |
| 119 | """ |
nothing calls this directly
no test coverage detected