MCPcopy
hub / github.com/scrapy/scrapy / parallel_async

Function parallel_async

scrapy/utils/defer.py:280–295  ·  view source on GitHub ↗

Like ``parallel`` but for async iterators

(
    async_iterable: AsyncIterator[_T],
    count: int,
    callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],  # noqa: A002
    *args: _P.args,
    **named: _P.kwargs,
)

Source from the content-addressed store, hash-verified

278
279
280def parallel_async(
281 async_iterable: AsyncIterator[_T],
282 count: int,
283 callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], # noqa: A002
284 *args: _P.args,
285 **named: _P.kwargs,
286) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]:
287 """Like ``parallel`` but for async iterators"""
288 coop = Cooperator()
289 work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter(
290 async_iterable, callable, *args, **named
291 )
292 dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList(
293 [coop.coiterate(work) for _ in range(count)]
294 )
295 return dl
296
297
298def process_chain(

Callers 3

test_simpleMethod · 0.90
test_delaysMethod · 0.90

Calls 1

Tested by 2

test_simpleMethod · 0.72
test_delaysMethod · 0.72