Function evaluate

examples/sldbench/evaluator.py:229–271

High-level, single-call evaluation function. This orchestrates the entire process: 1. Infers the task name. 2. Fits the model on training data. 3. Evaluates the fitted model on test data. 4. Returns a dictionary with final metrics and (optionally) fitted parameters. Ar

(program_path: str, verbose: bool = False)

        return get_failure_result(str(e))

def evaluate(program_path: str, verbose: bool = False) -> Dict[str, Any]:
    """
    High-level, single-call evaluation function.

    This orchestrates the entire process:
    1. Infers the task name.
    2. Fits the model on training data.
    3. Evaluates the fitted model on test data.
    4. Returns a dictionary with final metrics and (optionally) fitted parameters.

    Args:
        program_path: Path to the user's Python script with scaling law functions.
        verbose: If True, include fitted parameters and task name in the result.

    Returns:
        A dictionary containing the evaluation results.
    """
    try:
        task_name = resolve_task_name(program_path)
    except ValueError as e:
        return get_failure_result(str(e))

    # 1. Fit on training data to get parameters
    fit_result = evaluate_core(program_path, task_name, use_test_data=False)
    if "fitted_params" not in fit_result:
        error = fit_result.get("error", "Unknown fitting error.")
        return get_failure_result(f"Fitting failed: {error}")

    fitted_params_map = fit_result["fitted_params"]

    # 2. Evaluate on test data using the fitted parameters
    test_result = evaluate_core(
        program_path,
        task_name,
        use_test_data=True,
        fitted_params_map=fitted_params_map,
    )

    # 3. Combine results into a comprehensive output
    if verbose:
        test_result["fitted_params"] = fitted_params_map
        test_result["task_name"] = task_name
    return test_result

# --- Script Entrypoint ---