Function evaluate

examples/k_module_problem/evaluator.py:48–107

Evaluate a pipeline configuration program. Args: program_path: Path to the Python file containing configure_pipeline() Returns: dict with 'metrics' and optionally 'artifacts'

(program_path: str)

def evaluate(program_path: str) -> dict:
    """
    Evaluate a pipeline configuration program.

    Args:
        program_path: Path to the Python file containing configure_pipeline()

    Returns:
        dict with 'metrics' and optionally 'artifacts'
    """
    start_time = time.time()

    try:
        # Load and execute the program
        spec = importlib.util.spec_from_file_location("program", program_path)
        module = importlib.util.module_from_spec(spec)
        sys.modules["program"] = module
        spec.loader.exec_module(module)

        # Get the configuration
        if hasattr(module, 'run_pipeline'):
            config = module.run_pipeline()
        elif hasattr(module, 'configure_pipeline'):
            config = module.configure_pipeline()
        else:
            return _error_result("Program must define run_pipeline() or configure_pipeline()")

        # Validate the configuration
        validation_errors = validate_config(config)
        if validation_errors:
            return _validation_error_result(validation_errors)

        # Score the configuration
        correct_count, module_results = score_config(config)

        # Calculate metrics
        accuracy = correct_count / NUM_MODULES

        # The combined score rewards finding more correct modules
        # but gives NO information about which modules are correct
        combined_score = accuracy

        eval_time = time.time() - start_time

        # Build artifacts - provide feedback that helps evolution
        # but doesn't reveal which specific modules are wrong
        artifacts = build_artifacts(config, correct_count, module_results, eval_time)

        # Return metrics at top level for OpenEvolve compatibility
        return {
            "correct_modules": correct_count,
            "total_modules": NUM_MODULES,
            "accuracy": accuracy,
            "combined_score": combined_score,
            "eval_time": eval_time,
            "artifacts": artifacts,
        }