(
self, request: Request
)
| 483 | |
| 484 | @inlineCallbacks |
| 485 | def _download( |
| 486 | self, request: Request |
| 487 | ) -> Generator[Deferred[Any], Any, Response | Request]: |
| 488 | assert self._slot is not None # typing |
| 489 | assert self.spider is not None |
| 490 | |
| 491 | self._slot.add_request(request) |
| 492 | try: |
| 493 | result: Response | Request |
| 494 | if self._downloader_fetch_needs_spider: |
| 495 | result = yield self.downloader.fetch(request, self.spider) |
| 496 | else: |
| 497 | result = yield self.downloader.fetch(request) |
| 498 | if not isinstance(result, (Response, Request)): |
| 499 | raise TypeError( |
| 500 | f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" |
| 501 | ) |
| 502 | if isinstance(result, Response): |
| 503 | if result.request is None: |
| 504 | result.request = request |
| 505 | logkws = self.logformatter.crawled(result.request, result, self.spider) |
| 506 | if logkws is not None: |
| 507 | logger.log( |
| 508 | *logformatter_adapter(logkws), extra={"spider": self.spider} |
| 509 | ) |
| 510 | self.signals.send_catch_log( |
| 511 | signal=signals.response_received, |
| 512 | response=result, |
| 513 | request=result.request, |
| 514 | spider=self.spider, |
| 515 | ) |
| 516 | return result |
| 517 | finally: |
| 518 | self._slot.nextcall.schedule() |
| 519 | |
| 520 | def open_spider( |
| 521 | self, spider: Spider, close_if_idle: bool = True |
no test coverage detected