(self, gen_kwargs: dict)
| 97 | raise NotImplementedError() |
| 98 | |
| 99 | def _get_sampling_params(self, gen_kwargs: dict) -> SamplingParams: |
| 100 | params_mapping = { |
| 101 | "temperature": "temperature", |
| 102 | "top_p": "top_p", |
| 103 | "max_gen_toks": "max_tokens", |
| 104 | "until": "stop", |
| 105 | } |
| 106 | # IMPORTANT: |
| 107 | # lm-evaluation-harness controls generation primarily via per-task gen_kwargs. |
| 108 | # For example, the `local-completions` model wrapper uses: |
| 109 | # max_tokens <- gen_kwargs["max_tokens"] or gen_kwargs["max_gen_toks"] or _max_gen_toks |
| 110 | # temperature <- gen_kwargs.get("temperature", 0) |
| 111 | # stop <- gen_kwargs.get("until", ...) |
| 112 | # See: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py |
| 113 | |
| 114 | if self.sampling_params is None: |
| 115 | sampling_params = SamplingParams( |
| 116 | max_tokens=gen_kwargs.get("max_gen_toks", 256), |
| 117 | temperature=gen_kwargs.get("temperature", 0), |
| 118 | stop=gen_kwargs.get("until", None), |
| 119 | ) |
| 120 | else: |
| 121 | sampling_params = copy.deepcopy(self.sampling_params) |
| 122 | |
| 123 | for lm_eval_key, trtllm_key in params_mapping.items(): |
| 124 | value = gen_kwargs.pop(lm_eval_key, None) |
| 125 | if value is not None: |
| 126 | setattr(sampling_params, trtllm_key, value) |
| 127 | return sampling_params |
| 128 | |
| 129 | def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: |
| 130 | profiler.start("trtllm exec") |
no test coverage detected