Evaluate LongBench v2 benchmark. This overrides the base evaluate method to initialize tokenizer and chat template before evaluation, and handle CoT mode with two-pass inference. Args: llm: Language model instance sampling_params: Sampling parameters
(self,
llm: Any,
sampling_params: Optional[SamplingParams] = None,
streaming: bool = False)
| 337 | return prompt |
| 338 | |
| 339 | def evaluate(self, |
| 340 | llm: Any, |
| 341 | sampling_params: Optional[SamplingParams] = None, |
| 342 | streaming: bool = False) -> float: |
| 343 | """Evaluate LongBench v2 benchmark. |
| 344 | |
| 345 | This overrides the base evaluate method to initialize tokenizer and |
| 346 | chat template before evaluation, and handle CoT mode with two-pass inference. |
| 347 | |
| 348 | Args: |
| 349 | llm: Language model instance |
| 350 | sampling_params: Sampling parameters for generation |
| 351 | streaming: Whether to use streaming mode |
| 352 | |
| 353 | Returns: |
| 354 | Overall accuracy score |
| 355 | """ |
| 356 | # Initialize tokenizer and chat template |
| 357 | if hasattr(llm, 'tokenizer'): |
| 358 | self.tokenizer = llm.tokenizer |
| 359 | else: |
| 360 | logger.warning( |
| 361 | "LLM does not have tokenizer attribute. Truncation disabled.") |
| 362 | |
| 363 | # Store llm reference for CoT second pass |
| 364 | self.llm = llm |
| 365 | |
| 366 | # Call parent evaluate method |
| 367 | return super().evaluate(llm, sampling_params, streaming) |
| 368 | |
| 369 | def generate_samples(self) -> Iterable[tuple]: |
| 370 | """ |