Process everything in dependency order.
(graph: Graph, manager: BuildManager)
| 4553 | |
| 4554 | |
| 4555 | def process_graph(graph: Graph, manager: BuildManager) -> None: |
| 4556 | """Process everything in dependency order.""" |
| 4557 | if manager.workers: |
| 4558 | # Commit any cache writes from graph loading before workers try to read them. |
| 4559 | manager.commit() |
| 4560 | |
| 4561 | # Broadcast graph to workers before computing SCCs to save a bit of time. |
| 4562 | # TODO: check if we can optimize by sending only part of the graph needed for given SCC. |
| 4563 | # For example only send modules in the SCC and their dependencies. |
| 4564 | graph_message = GraphMessage(graph=graph, missing_modules=manager.missing_modules) |
| 4565 | buf = WriteBuffer() |
| 4566 | graph_message.write(buf) |
| 4567 | graph_data = buf.getvalue() |
| 4568 | manager.wait_ack() |
| 4569 | manager.broadcast(graph_data) |
| 4570 | |
| 4571 | sccs = sorted_components(graph) |
| 4572 | manager.log( |
| 4573 | "Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc.mod_ids) for scc in sccs)) |
| 4574 | ) |
| 4575 | scc_by_id = {scc.id: scc for scc in sccs} |
| 4576 | manager.scc_by_id = scc_by_id |
| 4577 | manager.top_order = [scc.id for scc in sccs] |
| 4578 | for scc in sccs: |
| 4579 | for mod_id in scc.mod_ids: |
| 4580 | manager.scc_by_mod_id[mod_id] = scc |
| 4581 | |
| 4582 | # Broadcast SCC structure to the parallel workers, since they don't compute it. |
| 4583 | sccs_message = SccsDataMessage(sccs=sccs) |
| 4584 | buf = WriteBuffer() |
| 4585 | sccs_message.write(buf) |
| 4586 | sccs_data = buf.getvalue() |
| 4587 | manager.wait_ack() |
| 4588 | manager.broadcast(sccs_data) |
| 4589 | manager.wait_ack() |
| 4590 | |
| 4591 | manager.free_workers = set(range(manager.options.num_workers)) |
| 4592 | |
| 4593 | # Prime the ready list with leaf SCCs (that have no dependencies). |
| 4594 | ready = [] |
| 4595 | not_ready = set() |
| 4596 | for scc in sccs: |
| 4597 | if not scc.deps: |
| 4598 | ready.append(scc) |
| 4599 | else: |
| 4600 | not_ready.add(scc) |
| 4601 | |
| 4602 | still_working = False |
| 4603 | while ready or not_ready or still_working: |
| 4604 | stale, fresh = find_stale_sccs(ready, graph, manager) |
| 4605 | if stale: |
| 4606 | for scc in stale: |
| 4607 | for id in scc.mod_ids: |
| 4608 | graph[id].mark_as_rechecked() |
| 4609 | manager.submit(graph, stale) |
| 4610 | still_working = True |
| 4611 | # We eagerly walk over fresh SCCs to reach as many stale SCCs as soon |
| 4612 | # as possible. Only when there are no fresh SCCs, we wait on scheduled stale ones. |
no test coverage detected
searching dependent graphs…