Run vLLM chat benchmark. This function is recommended ONLY for benchmarking multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead.
(
requests: list[SampleRequest], n: int, engine_args: EngineArgs, disable_detokenize: bool = False
)
| 79 | |
| 80 | |
| 81 | def run_fd_chat( |
| 82 | requests: list[SampleRequest], n: int, engine_args: EngineArgs, disable_detokenize: bool = False |
| 83 | ) -> tuple[float, list[RequestOutput]]: |
| 84 | """ |
| 85 | Run vLLM chat benchmark. This function is recommended ONLY for benchmarking |
| 86 | multimodal models as it properly handles multimodal inputs and chat |
| 87 | formatting. For non-multimodal models, use run_vllm() instead. |
| 88 | """ |
| 89 | from fastdeploy import LLM, SamplingParams |
| 90 | |
| 91 | llm = LLM(**dataclasses.asdict(engine_args)) |
| 92 | |
| 93 | assert all( |
| 94 | llm.llm_engine.cfg.max_model_len >= (request.prompt_len + request.expected_output_len) for request in requests |
| 95 | ), ( |
| 96 | "Please ensure that max_model_len is greater than the sum of " |
| 97 | "prompt_len and expected_output_len for all requests." |
| 98 | ) |
| 99 | |
| 100 | prompts = [] |
| 101 | sampling_params: list[SamplingParams] = [] |
| 102 | for request in requests: |
| 103 | prompts.append(request.prompt) |
| 104 | sampling_params.append( |
| 105 | SamplingParams( |
| 106 | n=n, |
| 107 | temperature=1.0, |
| 108 | top_p=1.0, |
| 109 | max_tokens=request.expected_output_len, |
| 110 | ) |
| 111 | ) |
| 112 | start = time.perf_counter() |
| 113 | outputs = llm.chat(prompts, sampling_params, use_tqdm=True) |
| 114 | end = time.perf_counter() |
| 115 | return end - start, outputs |
| 116 | |
| 117 | |
| 118 | def run_hf( |