(ctx, **kwargs)
| 974 | @click.pass_context |
| 975 | @staticmethod |
| 976 | def command(ctx, **kwargs) -> None: |
| 977 | llm: Union[LLM, PyTorchLLM] = ctx.obj |
| 978 | |
| 979 | evaluator = LongBenchV1( |
| 980 | dataset_path=kwargs.pop("dataset_path", None), |
| 981 | num_samples=kwargs.pop("num_samples", None), |
| 982 | random_seed=kwargs.pop("random_seed", 0), |
| 983 | apply_chat_template=kwargs.pop("apply_chat_template", True), |
| 984 | system_prompt=kwargs.pop("system_prompt", None), |
| 985 | chat_template_kwargs=kwargs.pop("chat_template_kwargs", None), |
| 986 | log_samples=kwargs.pop("log_samples", False), |
| 987 | output_path=kwargs.pop("output_path", None)) |
| 988 | |
| 989 | # Let lm-eval task configs control sampling via gen_kwargs. |
| 990 | sampling_params = None |
| 991 | |
| 992 | evaluator.evaluate(llm, sampling_params) |
| 993 | llm.shutdown() |
nothing calls this directly
no test coverage detected