Function evaluate

examples/signal_processing/evaluator.py:266–453

Main evaluation function that tests the signal processing algorithm on multiple test signals and calculates the composite performance metric.

(program_path)

def evaluate(program_path):
    """
    Main evaluation function that tests the signal processing algorithm
    on multiple test signals and calculates the composite performance metric.
    """
    try:
        # Load the program
        spec = importlib.util.spec_from_file_location("program", program_path)
        program = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(program)

        # Check if required function exists
        if not hasattr(program, "run_signal_processing"):
            return {"composite_score": 0.0, "error": "Missing run_signal_processing function"}

        # Generate test signals
        test_signals = generate_test_signals(5)

        # Collect metrics across all test signals
        all_scores = []
        all_metrics = []
        successful_runs = 0

        for i, (noisy_signal, clean_signal) in enumerate(test_signals):
            try:
                # Run the algorithm with timeout
                start_time = time.time()

                # Call the program's main function
                result = run_with_timeout(
                    program.run_signal_processing,
                    kwargs={
                        "signal_length": len(noisy_signal),
                        "noise_level": 0.3,
                        "window_size": 20,
                    },
                    timeout_seconds=10,
                )

                execution_time = time.time() - start_time

                # Validate result format
                if not isinstance(result, dict):
                    print(f"Signal {i}: Invalid result format")
                    continue

                if "filtered_signal" not in result:
                    print(f"Signal {i}: Missing filtered_signal in result")
                    continue

                filtered_signal = result["filtered_signal"]

                if len(filtered_signal) == 0:
                    print(f"Signal {i}: Empty filtered signal")
                    continue

                # Convert to numpy arrays
                filtered_signal = np.array(filtered_signal)