Starts consuming Spider.start() output and sending scheduled requests.
(self)
| 297 | self._slot.nextcall.schedule() |
| 298 | |
| 299 | async def _start_request_processing(self) -> None: |
| 300 | """Starts consuming Spider.start() output and sending scheduled |
| 301 | requests.""" |
| 302 | # Starts the processing of scheduled requests, as well as a periodic |
| 303 | # call to that processing method for scenarios where the scheduler |
| 304 | # reports having pending requests but returns none. |
| 305 | try: |
| 306 | assert self._slot is not None # typing |
| 307 | self._slot.nextcall.schedule() |
| 308 | self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) |
| 309 | |
| 310 | while self._start and self.spider and self.running: |
| 311 | await self._process_start_next() |
| 312 | if not self.needs_backout(): |
| 313 | # Give room for the outcome of self._process_start_next() to be |
| 314 | # processed before continuing with the next iteration. |
| 315 | self._slot.nextcall.schedule() |
| 316 | await self._slot.nextcall.wait() |
| 317 | except (asyncio.exceptions.CancelledError, CancelledError): |
| 318 | # self.stop_async() has cancelled us, nothing to do |
| 319 | return |
| 320 | except Exception: |
| 321 | # an error happened, log it and stop the engine |
| 322 | self._start_request_processing_awaitable = None |
| 323 | logger.error( |
| 324 | "Error while processing requests from start()", |
| 325 | exc_info=True, |
| 326 | extra={"spider": self.spider}, |
| 327 | ) |
| 328 | await self.stop_async() |
| 329 | |
| 330 | def _start_scheduled_requests(self) -> None: |
| 331 | if self._slot is None or self._slot.closing is not None or self.paused: |
no test coverage detected