(self, slot: Slot, request: Request)
| 221 | self._process_queue(slot) |
| 222 | |
| 223 | async def _download(self, slot: Slot, request: Request) -> Response: |
| 224 | # The order is very important for the following logic. Do not change! |
| 225 | slot.transferring.add(request) |
| 226 | try: |
| 227 | # 1. Download the response |
| 228 | response: Response = await self.handlers.download_request_async(request) |
| 229 | # 2. Notify response_downloaded listeners about the recent download |
| 230 | # before querying queue for next request |
| 231 | self.signals.send_catch_log( |
| 232 | signal=signals.response_downloaded, |
| 233 | response=response, |
| 234 | request=request, |
| 235 | spider=self.crawler.spider, |
| 236 | ) |
| 237 | return response |
| 238 | except Exception: |
| 239 | await _defer_sleep_async() |
| 240 | raise |
| 241 | finally: |
| 242 | # 3. After response arrives, remove the request from transferring |
| 243 | # state to free up the transferring slot so it can be used by the |
| 244 | # following requests (perhaps those which came from the downloader |
| 245 | # middleware itself) |
| 246 | slot.transferring.remove(request) |
| 247 | self._process_queue(slot) |
| 248 | self.signals.send_catch_log( |
| 249 | signal=signals.request_left_downloader, |
| 250 | request=request, |
| 251 | spider=self.crawler.spider, |
| 252 | ) |
| 253 | |
| 254 | async def _wait_for_download( |
| 255 | self, slot: Slot, request: Request, queue_dfd: Deferred[Response] |
no test coverage detected