(self, requests, disable_tqdm: bool = False)
| 127 | return sampling_params |
| 128 | |
| 129 | def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: |
| 130 | profiler.start("trtllm exec") |
| 131 | results = [] |
| 132 | for request in tqdm(requests, |
| 133 | desc="Submitting requests", |
| 134 | disable=disable_tqdm): |
| 135 | prompt, gen_kwargs = request.args |
| 136 | sampling_params = self._get_sampling_params(gen_kwargs) |
| 137 | output = self.llm.generate_async(prompt, |
| 138 | sampling_params=sampling_params, |
| 139 | streaming=self.streaming) |
| 140 | results.append(output) |
| 141 | |
| 142 | outputs = [] |
| 143 | for output in tqdm(results, |
| 144 | desc="Fetching responses", |
| 145 | disable=disable_tqdm): |
| 146 | outputs.append(output.result()) |
| 147 | |
| 148 | profiler.stop("trtllm exec") |
| 149 | elapsed_time = profiler.elapsed_time_in_sec("trtllm exec") |
| 150 | logger.info(f"TRTLLM execution time: {elapsed_time:.3f} seconds.") |
| 151 | profiler.reset("trtllm exec") |
| 152 | |
| 153 | return [output.outputs[0].text for output in outputs] |
| 154 | |
| 155 | |
| 156 | class MultimodalLmEvalWrapper(LmEvalWrapper): |
nothing calls this directly
no test coverage detected