Core evaluation logic: fits a model or evaluates it on test data.
(
program_path: str,
task_name: str,
use_test_data: bool = False,
fitted_params_map: Dict[Any, Any] = None,
)
| 166 | # --- Evaluation Pipelines --- |
| 167 | |
| 168 | def evaluate_core( |
| 169 | program_path: str, |
| 170 | task_name: str, |
| 171 | use_test_data: bool = False, |
| 172 | fitted_params_map: Dict[Any, Any] = None, |
| 173 | ) -> Dict[str, Union[float, Dict]]: |
| 174 | """ |
| 175 | Core evaluation logic: fits a model or evaluates it on test data. |
| 176 | """ |
| 177 | try: |
| 178 | program = _import_program(program_path) |
| 179 | fit_scaling_law = program.fit_scaling_law |
| 180 | scaling_law_func = program.scaling_law_func |
| 181 | |
| 182 | if not use_test_data: |
| 183 | # --- FIT on training data --- |
| 184 | train_data = load_data(task_name, train=True) |
| 185 | if not train_data: |
| 186 | return get_failure_result("No training data found.") |
| 187 | |
| 188 | new_fitted_params_map = {} |
| 189 | for key, (X_train, y_train) in train_data.items(): |
| 190 | params = run_with_timeout(fit_scaling_law, args=(X_train, y_train)) |
| 191 | new_fitted_params_map[key] = params |
| 192 | return {"fitted_params": new_fitted_params_map} |
| 193 | |
| 194 | else: |
| 195 | # --- EVALUATE on test data --- |
| 196 | if fitted_params_map is None: |
| 197 | return get_failure_result("fitted_params_map is required for evaluation.") |
| 198 | |
| 199 | test_data = load_data(task_name, train=False) |
| 200 | if not test_data: |
| 201 | return get_failure_result("No test data found.") |
| 202 | |
| 203 | all_predictions, all_true_values = [], [] |
| 204 | for key, (X_test, y_test) in test_data.items(): |
| 205 | if key not in fitted_params_map: |
| 206 | print(f"Warning: No params for test group '{key}'. Skipping.", file=sys.stderr) |
| 207 | continue |
| 208 | |
| 209 | params = fitted_params_map[key] |
| 210 | predictions = run_with_timeout(scaling_law_func, args=(X_test, params)) |
| 211 | all_predictions.append(np.asarray(predictions)) |
| 212 | all_true_values.append(np.asarray(y_test)) |
| 213 | |
| 214 | if not all_predictions: |
| 215 | return get_failure_result("No predictions were generated for the test set.") |
| 216 | |
| 217 | final_predictions = np.concatenate(all_predictions) |
| 218 | final_true_values = np.concatenate(all_true_values) |
| 219 | |
| 220 | return calculate_final_metrics( |
| 221 | final_predictions, |
| 222 | final_true_values, |
| 223 | ) |
| 224 | |
| 225 | except Exception as e: |
no test coverage detected