Benchmarks an API endpoint using a given set of sample inputs and returns
(
backend: str,
api_url: str,
base_url: str,
model_id: str,
model_name: str,
input_requests: list[SampleRequest],
hyper_parameters: dict,
logprobs: Optional[int],
request_rate: float,
burstiness: float,
disable_tqdm: bool,
profile: bool,
selected_percentile_metrics: list[str],
selected_percentiles: list[float],
ignore_eos: bool,
debug: bool,
goodput_config_dict: dict[str, float],
max_concurrency: Optional[int],
lora_modules: Optional[Iterable[str]],
extra_body: Optional[dict],
)
| 605 | |
| 606 | |
| 607 | async def benchmark( |
| 608 | backend: str, |
| 609 | api_url: str, |
| 610 | base_url: str, |
| 611 | model_id: str, |
| 612 | model_name: str, |
| 613 | input_requests: list[SampleRequest], |
| 614 | hyper_parameters: dict, |
| 615 | logprobs: Optional[int], |
| 616 | request_rate: float, |
| 617 | burstiness: float, |
| 618 | disable_tqdm: bool, |
| 619 | profile: bool, |
| 620 | selected_percentile_metrics: list[str], |
| 621 | selected_percentiles: list[float], |
| 622 | ignore_eos: bool, |
| 623 | debug: bool, |
| 624 | goodput_config_dict: dict[str, float], |
| 625 | max_concurrency: Optional[int], |
| 626 | lora_modules: Optional[Iterable[str]], |
| 627 | extra_body: Optional[dict], |
| 628 | ): |
| 629 | """Benchmarks an API endpoint using a given set of sample inputs and returns""" |
| 630 | if backend in ASYNC_REQUEST_FUNCS: |
| 631 | request_func = ASYNC_REQUEST_FUNCS[backend] |
| 632 | else: |
| 633 | raise ValueError(f"Unknown backend: {backend}") |
| 634 | |
| 635 | print("Starting initial single prompt test run...") |
| 636 | test_prompt, test_output_len, test_no = ( |
| 637 | input_requests[0].prompt, |
| 638 | input_requests[0].expected_output_len, |
| 639 | input_requests[0].no, |
| 640 | ) |
| 641 | test_history_QA = input_requests[0].history_QA |
| 642 | |
| 643 | test_input = RequestFuncInput( |
| 644 | model=model_id, |
| 645 | model_name=model_name, |
| 646 | prompt=test_prompt, |
| 647 | no=test_no, |
| 648 | prompt_len=0, |
| 649 | history_QA=test_history_QA, |
| 650 | hyper_parameters=hyper_parameters, |
| 651 | api_url=api_url, |
| 652 | output_len=test_output_len, |
| 653 | logprobs=logprobs, |
| 654 | ignore_eos=ignore_eos, |
| 655 | debug=debug, |
| 656 | extra_body=extra_body, |
| 657 | ) |
| 658 | |
| 659 | print("test_input:", test_input) |
| 660 | |
| 661 | test_output = await request_func(request_func_input=test_input) |
| 662 | |
| 663 | print("test_output:", test_output) |
| 664 |