(self, crawler: Crawler)
| 101 | _SLOT_GC_INTERVAL: float = 60.0 # seconds |
| 102 | |
| 103 | def __init__(self, crawler: Crawler): |
| 104 | self.crawler: Crawler = crawler |
| 105 | self.settings: BaseSettings = crawler.settings |
| 106 | self.signals: SignalManager = crawler.signals |
| 107 | self.slots: dict[str, Slot] = {} |
| 108 | self.active: set[Request] = set() |
| 109 | self.handlers: DownloadHandlers = DownloadHandlers(crawler) |
| 110 | self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") |
| 111 | self.domain_concurrency: int = self.settings.getint( |
| 112 | "CONCURRENT_REQUESTS_PER_DOMAIN" |
| 113 | ) |
| 114 | self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") |
| 115 | self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") |
| 116 | self.middleware: DownloaderMiddlewareManager = ( |
| 117 | DownloaderMiddlewareManager.from_crawler(crawler) |
| 118 | ) |
| 119 | self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None |
| 120 | self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( |
| 121 | "DOWNLOAD_SLOTS" |
| 122 | ) |
| 123 | |
| 124 | @inlineCallbacks |
| 125 | @_warn_spider_arg |
nothing calls this directly
no test coverage detected