Define delay adjustment policy
(self, slot: Slot, latency: float, response: Response)
| 102 | return key, self.crawler.engine.downloader.slots.get(key) |
| 103 | |
| 104 | def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: |
| 105 | """Define delay adjustment policy""" |
| 106 | |
| 107 | # If a server needs `latency` seconds to respond then |
| 108 | # we should send a request each `latency/N` seconds |
| 109 | # to have N requests processed in parallel |
| 110 | target_delay = latency / self.target_concurrency |
| 111 | |
| 112 | # Adjust the delay to make it closer to target_delay |
| 113 | new_delay = (slot.delay + target_delay) / 2.0 |
| 114 | |
| 115 | # If target delay is bigger than old delay, then use it instead of mean. |
| 116 | # It works better with problematic sites. |
| 117 | new_delay = max(target_delay, new_delay) |
| 118 | |
| 119 | # Make sure self.mindelay <= new_delay <= self.max_delay |
| 120 | new_delay = min(max(self.mindelay, new_delay), self.maxdelay) |
| 121 | |
| 122 | # Dont adjust delay if response status != 200 and new delay is smaller |
| 123 | # than old one, as error pages (and redirections) are usually small and |
| 124 | # so tend to reduce latency, thus provoking a positive feedback by |
| 125 | # reducing delay instead of increase. |
| 126 | if response.status != 200 and new_delay <= slot.delay: |
| 127 | return |
| 128 | |
| 129 | slot.delay = new_delay |
no outgoing calls
no test coverage detected