MCPcopy
hub / github.com/scrapy/scrapy / _process_queue

Method _process_queue

scrapy/core/downloader/__init__.py:195–217  ·  view source on GitHub ↗
(self, slot: Slot)

Source from the content-addressed store, hash-verified

193 slot.active.remove(request)
194
195 def _process_queue(self, slot: Slot) -> None:
196 if slot.latercall:
197 # block processing until slot.latercall is called
198 return
199
200 # Delay queue processing if a download_delay is configured
201 now = monotonic()
202 delay = slot.download_delay()
203 if delay:
204 penalty = delay - now + slot.lastseen
205 if penalty > 0:
206 slot.latercall = call_later(penalty, self._latercall, slot)
207 return
208
209 # Process enqueued requests if there are free slots to transfer for this slot
210 while slot.queue and slot.free_transfer_slots() > 0:
211 slot.lastseen = now
212 request, queue_dfd = slot.queue.popleft()
213 _schedule_coro(self._wait_for_download(slot, request, queue_dfd))
214 # prevent burst if inter-request delays were configured
215 if delay:
216 self._process_queue(slot)
217 break
218
219 def _latercall(self, slot: Slot) -> None:
220 slot.latercall = None

Callers 3

_enqueue_requestMethod · 0.95
_latercallMethod · 0.95
_downloadMethod · 0.95

Calls 5

_wait_for_downloadMethod · 0.95
call_laterFunction · 0.90
_schedule_coroFunction · 0.90
download_delayMethod · 0.80
free_transfer_slotsMethod · 0.80

Tested by

no test coverage detected