Main evaluator class for SimpleQA benchmark
| 82 | |
| 83 | |
| 84 | class SimpleQAEvaluator: |
| 85 | """Main evaluator class for SimpleQA benchmark""" |
| 86 | |
| 87 | def __init__(self, |
| 88 | model: str, |
| 89 | approach: str, |
| 90 | base_url: str = DEFAULT_BASE_URL, |
| 91 | grader_model: str = DEFAULT_GRADER_MODEL, |
| 92 | timeout: int = DEFAULT_TIMEOUT, |
| 93 | cache_dir: str = "cache", |
| 94 | output_dir: str = "results", |
| 95 | use_verified: bool = False): |
| 96 | self.model = model |
| 97 | self.approach = approach |
| 98 | self.base_url = base_url |
| 99 | self.grader_model = grader_model |
| 100 | self.timeout = timeout |
| 101 | self.use_verified = use_verified |
| 102 | self.cache_dir = Path(cache_dir) |
| 103 | self.output_dir = Path(output_dir) |
| 104 | |
| 105 | # Create directories |
| 106 | self.cache_dir.mkdir(exist_ok=True) |
| 107 | self.output_dir.mkdir(exist_ok=True) |
| 108 | |
| 109 | # Setup OptILLM client with extended timeout |
| 110 | self.optillm_client = OpenAI( |
| 111 | api_key="optillm", |
| 112 | base_url=base_url, |
| 113 | timeout=httpx.Timeout(timeout, connect=5.0), |
| 114 | max_retries=0 |
| 115 | ) |
| 116 | |
| 117 | # Setup grader client (use OptILLM for grading) |
| 118 | try: |
| 119 | self.grader_client = OpenAI( |
| 120 | api_key="optillm", |
| 121 | base_url=base_url, |
| 122 | timeout=httpx.Timeout(timeout, connect=5.0), |
| 123 | max_retries=0 |
| 124 | ) |
| 125 | logger.info("Using OptILLM for grading responses") |
| 126 | except Exception as e: |
| 127 | logger.warning(f"Could not initialize grader client: {e}") |
| 128 | logger.warning("Grading will be skipped.") |
| 129 | self.grader_client = None |
| 130 | |
| 131 | # Results tracking |
| 132 | self.results = [] |
| 133 | self.metrics = { |
| 134 | "correct": 0, |
| 135 | "incorrect": 0, |
| 136 | "not_attempted": 0, |
| 137 | "errors": 0, |
| 138 | "total_processed": 0 |
| 139 | } |
| 140 | |
| 141 | def download_dataset(self) -> str: |