Method run

agent/src/opentrace_agent/benchmarks/swe_bench.py:384–462

Run the full SWE-bench evaluation. Parameters ---------- instances_path : str or Path Path to SWE-bench instances JSON file. agent_fn : AgentFn The agent function to test. use_opentrace : bool Whether to provide OpenTrace t

(
        self,
        instances_path: str | Path,
        agent_fn: AgentFn,
        *,
        use_opentrace: bool = True,
        limit: int | None = None,
        on_progress: Callable[[int, int, SWEBenchResult], None] | None = None,
        workers: int = 1,
    )

        return result

    def run(
        self,
        instances_path: str | Path,
        agent_fn: AgentFn,
        *,
        use_opentrace: bool = True,
        limit: int | None = None,
        on_progress: Callable[[int, int, SWEBenchResult], None] | None = None,
        workers: int = 1,
    ) -> SWEBenchReport:
        """Run the full SWE-bench evaluation.

        Parameters
        ----------
        instances_path : str or Path
            Path to SWE-bench instances JSON file.
        agent_fn : AgentFn
            The agent function to test.
        use_opentrace : bool
            Whether to provide OpenTrace tools.
        limit : int or None
            Max instances to run (for quick testing).
        on_progress : callable or None
            Called after each instance with ``(completed, total, result)``.
        workers : int
            Number of instances to run in parallel (default 1 = sequential).
        """
        instances = self.load_instances(instances_path)
        if limit:
            instances = instances[:limit]

        t0 = time.monotonic()
        total = len(instances)

        if workers <= 1:
            # Sequential
            results = []
            for i, instance in enumerate(instances):
                logger.info(
                    "[%d/%d] Running %s (opentrace=%s)",
                    i + 1,
                    total,
                    instance.instance_id,
                    use_opentrace,
                )
                result = self.run_instance(instance, agent_fn, use_opentrace=use_opentrace)
                results.append(result)
                if on_progress is not None:
                    on_progress(i + 1, total, result)
        else:
            # Parallel with thread pool
            from concurrent.futures import ThreadPoolExecutor, as_completed

            results = [None] * total  # type: ignore[list-item]
            completed = 0

            def _run(idx: int, inst: Any) -> tuple[int, SWEBenchResult]:
                logger.info(