Evaluate a program and return scores Args: program_code: Code to evaluate program_id: Optional ID for logging Returns: Dictionary of metric name to score
(
self,
program_code: str,
program_id: str = "",
)
| 130 | ) |
| 131 | |
| 132 | async def evaluate_program( |
| 133 | self, |
| 134 | program_code: str, |
| 135 | program_id: str = "", |
| 136 | ) -> Dict[str, float]: |
| 137 | """ |
| 138 | Evaluate a program and return scores |
| 139 | |
| 140 | Args: |
| 141 | program_code: Code to evaluate |
| 142 | program_id: Optional ID for logging |
| 143 | |
| 144 | Returns: |
| 145 | Dictionary of metric name to score |
| 146 | """ |
| 147 | start_time = time.time() |
| 148 | program_id_str = f" {program_id}" if program_id else "" |
| 149 | |
| 150 | # Check if artifacts are enabled |
| 151 | artifacts_enabled = os.environ.get("ENABLE_ARTIFACTS", "true").lower() == "true" |
| 152 | |
| 153 | # Retry logic for evaluation |
| 154 | last_exception = None |
| 155 | for attempt in range(self.config.max_retries + 1): |
| 156 | # Create a temporary file for the program |
| 157 | with tempfile.NamedTemporaryFile(suffix=self.program_suffix, delete=False) as temp_file: |
| 158 | temp_file.write(program_code.encode("utf-8")) |
| 159 | temp_file_path = temp_file.name |
| 160 | |
| 161 | try: |
| 162 | # Run evaluation |
| 163 | if self.config.cascade_evaluation: |
| 164 | # Run cascade evaluation |
| 165 | result = await self._cascade_evaluate(temp_file_path) |
| 166 | else: |
| 167 | # Run direct evaluation |
| 168 | result = await self._direct_evaluate(temp_file_path) |
| 169 | |
| 170 | # Process the result based on type |
| 171 | eval_result = self._process_evaluation_result(result) |
| 172 | |
| 173 | # Check if this was a timeout and capture artifacts if enabled |
| 174 | if artifacts_enabled and program_id and eval_result.metrics.get("timeout") is True: |
| 175 | if program_id not in self._pending_artifacts: |
| 176 | self._pending_artifacts[program_id] = {} |
| 177 | |
| 178 | self._pending_artifacts[program_id].update( |
| 179 | { |
| 180 | "timeout": True, |
| 181 | "timeout_duration": self.config.timeout, |
| 182 | "failure_stage": "evaluation", |
| 183 | "error_type": "timeout", |
| 184 | } |
| 185 | ) |
| 186 | |
| 187 | # Add LLM feedback if configured |
| 188 | llm_eval_result = None |
| 189 | if self.config.use_llm_feedback and self.llm_ensemble: |