Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel. .. versionadded:: 2.13
(
self,
result: Iterable[_T] | AsyncIterator[_T],
request: Request,
response: Response | Failure,
)
| 397 | ) |
| 398 | |
| 399 | async def handle_spider_output_async( |
| 400 | self, |
| 401 | result: Iterable[_T] | AsyncIterator[_T], |
| 402 | request: Request, |
| 403 | response: Response | Failure, |
| 404 | ) -> None: |
| 405 | """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel. |
| 406 | |
| 407 | .. versionadded:: 2.13 |
| 408 | """ |
| 409 | it: Iterable[_T] | AsyncIterator[_T] |
| 410 | if is_asyncio_available(): |
| 411 | if isinstance(result, AsyncIterator): |
| 412 | it = aiter_errback(result, self.handle_spider_error, request, response) |
| 413 | else: |
| 414 | it = iter_errback(result, self.handle_spider_error, request, response) |
| 415 | await _parallel_asyncio( |
| 416 | it, self.concurrent_items, self._process_spidermw_output_async, response |
| 417 | ) |
| 418 | return |
| 419 | if isinstance(result, AsyncIterator): |
| 420 | it = aiter_errback(result, self.handle_spider_error, request, response) |
| 421 | await maybe_deferred_to_future( |
| 422 | parallel_async( |
| 423 | it, |
| 424 | self.concurrent_items, |
| 425 | self._process_spidermw_output, |
| 426 | response, |
| 427 | ) |
| 428 | ) |
| 429 | return |
| 430 | it = iter_errback(result, self.handle_spider_error, request, response) |
| 431 | await maybe_deferred_to_future( |
| 432 | parallel( |
| 433 | it, |
| 434 | self.concurrent_items, |
| 435 | self._process_spidermw_output, |
| 436 | response, |
| 437 | ) |
| 438 | ) |
| 439 | |
| 440 | def _process_spidermw_output( |
| 441 | self, output: Any, response: Response | Failure |
no test coverage detected