| 34 | |
| 35 | |
| 36 | def run_fd( |
| 37 | requests: list[SampleRequest], |
| 38 | n: int, |
| 39 | engine_args: EngineArgs, |
| 40 | disable_detokenize: bool = False, |
| 41 | ) -> tuple[float, Optional[list[RequestOutput]]]: |
| 42 | from fastdeploy import LLM, SamplingParams |
| 43 | |
| 44 | llm = LLM(**dataclasses.asdict(engine_args)) |
| 45 | assert all( |
| 46 | llm.llm_engine.cfg.max_model_len >= (request.prompt_len + request.expected_output_len) for request in requests |
| 47 | ), ( |
| 48 | "Please ensure that max_model_len is greater than the sum of" |
| 49 | " prompt_len and expected_output_len for all requests." |
| 50 | ) |
| 51 | # Add the requests to the engine. |
| 52 | prompts = [] |
| 53 | sampling_params: list[SamplingParams] = [] |
| 54 | for request in requests: |
| 55 | # 处理tokenized输入 |
| 56 | if "prompt_token_ids" in request.prompt: |
| 57 | prompt = { |
| 58 | "prompt_token_ids": request.prompt["prompt_token_ids"], |
| 59 | "multi_modal_data": getattr(request, "multi_modal_data", None), |
| 60 | } |
| 61 | # 处理普通文本输入 |
| 62 | else: |
| 63 | prompt = {"prompt": str(request.prompt), "multi_modal_data": getattr(request, "multi_modal_data", None)} |
| 64 | prompts.append(prompt) |
| 65 | |
| 66 | sampling_params.append( |
| 67 | SamplingParams( |
| 68 | n=n, |
| 69 | temperature=1.0, |
| 70 | top_p=1.0, |
| 71 | max_tokens=request.expected_output_len, |
| 72 | ) |
| 73 | ) |
| 74 | outputs = None |
| 75 | start = time.perf_counter() |
| 76 | outputs = llm.generate(prompts, sampling_params, use_tqdm=True) |
| 77 | end = time.perf_counter() |
| 78 | return end - start, outputs |
| 79 | |
| 80 | |
| 81 | def run_fd_chat( |