Method get_scc_batch

mypy/build.py:1411–1431

Get a batch of SCCs from queue to submit to a worker. We batch SCCs to avoid communication overhead, but to avoid long poles, we limit fraction of work per worker.

(self, max_size_in_batch: int)

            self.scc_queue.extend([(0, 0, scc) for scc in sccs])

    def get_scc_batch(self, max_size_in_batch: int) -> list[SCC]:
        """Get a batch of SCCs from queue to submit to a worker.

        We batch SCCs to avoid communication overhead, but to avoid
        long poles, we limit fraction of work per worker.
        """
        batch: list[SCC] = []
        size_in_batch = 0
        while self.scc_queue and (
            # Three notes keep in mind here:
            #   * Heap key is *negative* size (so that larger SCCs appear first).
            #   * Each batch must have at least one item.
            #   * Adding another SCC to batch should not exceed maximum allowed size.
            size_in_batch - self.scc_queue[0][0] <= max_size_in_batch
            or not batch
        ):
            size_key, _, scc = heappop(self.scc_queue)
            size_in_batch -= size_key
            self.size_in_queue += size_key
            batch.append(scc)
        return batch

    def max_batch_size(self) -> int:
        batch_frac = 1 / len(self.workers)