MCPcopy
hub / github.com/scrapy/scrapy / _start_scheduled_request

Method _start_scheduled_request

scrapy/core/engine.py:360–396  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

358 self._slot.remove_request(request)
359
360 def _start_scheduled_request(self) -> bool:
361 assert self._slot is not None # typing
362 assert self.spider is not None # typing
363
364 request = self._slot.scheduler.next_request()
365 if request is None:
366 self.signals.send_catch_log(signals.scheduler_empty)
367 return False
368
369 d: Deferred[Response | Request] = self._download(request)
370 d.addBoth(self._handle_downloader_output, request)
371 d.addErrback(
372 lambda f: logger.info(
373 "Error while handling downloader output",
374 exc_info=failure_to_exc_info(f),
375 extra={"spider": self.spider},
376 )
377 )
378
379 d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request))
380 d2.addErrback(
381 lambda f: logger.info(
382 "Error while removing request from slot",
383 exc_info=failure_to_exc_info(f),
384 extra={"spider": self.spider},
385 )
386 )
387 slot = self._slot
388 d2.addBoth(lambda _: slot.nextcall.schedule())
389 d2.addErrback(
390 lambda f: logger.info(
391 "Error while scheduling new request",
392 exc_info=failure_to_exc_info(f),
393 extra={"spider": self.spider},
394 )
395 )
396 return True
397
398 @inlineCallbacks
399 def _handle_downloader_output(

Callers 1

Calls 6

_downloadMethod · 0.95
failure_to_exc_infoFunction · 0.90
send_catch_logMethod · 0.80
infoMethod · 0.80
scheduleMethod · 0.80
next_requestMethod · 0.45

Tested by

no test coverage detected