Handle the downloaded response or failure through the spider callback/errback.
(self, result: Response | Failure, request: Request)
| 242 | _schedule_coro(self._wait_for_processing(result, request, queue_dfd)) |
| 243 | |
| 244 | async def _scrape(self, result: Response | Failure, request: Request) -> None: |
| 245 | """Handle the downloaded response or failure through the spider callback/errback.""" |
| 246 | if not isinstance(result, (Response, Failure)): |
| 247 | raise TypeError( |
| 248 | f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" |
| 249 | ) |
| 250 | |
| 251 | output: Iterable[Any] | AsyncIterator[Any] |
| 252 | if isinstance(result, Response): |
| 253 | try: |
| 254 | # call the spider middlewares and the request callback with the response |
| 255 | output = await self.spidermw.scrape_response_async( |
| 256 | self.call_spider_async, result, request |
| 257 | ) |
| 258 | except Exception: |
| 259 | self.handle_spider_error(Failure(), request, result) |
| 260 | else: |
| 261 | await self.handle_spider_output_async(output, request, result) |
| 262 | return |
| 263 | |
| 264 | try: |
| 265 | # call the request errback with the downloader error |
| 266 | output = await self.call_spider_async(result, request) |
| 267 | except Exception as spider_exc: |
| 268 | # the errback didn't silence the exception |
| 269 | assert self.crawler.spider |
| 270 | if not result.check(IgnoreRequest): |
| 271 | logkws = self.logformatter.download_error( |
| 272 | result, request, self.crawler.spider |
| 273 | ) |
| 274 | logger.log( |
| 275 | *logformatter_adapter(logkws), |
| 276 | extra={"spider": self.crawler.spider}, |
| 277 | exc_info=failure_to_exc_info(result), |
| 278 | ) |
| 279 | if spider_exc is not result.value: |
| 280 | # the errback raised a different exception, handle it |
| 281 | self.handle_spider_error(Failure(), request, result) |
| 282 | else: |
| 283 | await self.handle_spider_output_async(output, request, result) |
| 284 | |
| 285 | async def _wait_for_processing( |
| 286 | self, result: Response | Failure, request: Request, queue_dfd: Deferred[None] |
no test coverage detected