(self,
llm: Any,
sampling_params: Optional[SamplingParams] = None,
streaming: bool = False)
| 83 | return sampling_params |
| 84 | |
| 85 | def evaluate(self, |
| 86 | llm: Any, |
| 87 | sampling_params: Optional[SamplingParams] = None, |
| 88 | streaming: bool = False) -> float: |
| 89 | profiler.start("trtllm exec") |
| 90 | outputs, references, auxiliaries = [], [], [] |
| 91 | for prompt, sampling_args, reference, *aux in tqdm( |
| 92 | self.generate_samples(), desc="Submitting requests"): |
| 93 | if self.apply_chat_template: |
| 94 | prompt = self.do_apply_chat_template(llm, prompt) |
| 95 | sampling_params = self._get_sampline_params(sampling_params, |
| 96 | sampling_args) |
| 97 | output = llm.generate_async( |
| 98 | prompt, |
| 99 | sampling_params, |
| 100 | streaming=streaming, |
| 101 | ) |
| 102 | outputs.append(output) |
| 103 | references.append(reference) |
| 104 | auxiliaries.append(aux) |
| 105 | results = [] |
| 106 | for output in tqdm(outputs, desc="Fetching responses"): |
| 107 | results.append(output.result()) |
| 108 | profiler.stop("trtllm exec") |
| 109 | elapsed_time = profiler.elapsed_time_in_sec("trtllm exec") |
| 110 | logger.info(f"TRTLLM execution time: {elapsed_time:.3f} seconds.") |
| 111 | profiler.reset("trtllm exec") |
| 112 | |
| 113 | score = self.compute_score(results, references, *zip(*auxiliaries)) |
| 114 | return score |
| 115 | |
| 116 | @staticmethod |
| 117 | def command(ctx, *args, **kwargs) -> None: |
nothing calls this directly
no test coverage detected