(
self, request: Request, info: SpiderInfo, item: Any
)
| 149 | return self.item_completed(results, item, info) |
| 150 | |
| 151 | async def _process_request( |
| 152 | self, request: Request, info: SpiderInfo, item: Any |
| 153 | ) -> FileInfo: |
| 154 | fp = self._fingerprinter.fingerprint(request) |
| 155 | |
| 156 | eb: Callable[[Failure], FileInfo] | None = request.errback |
| 157 | request.callback = NO_CALLBACK |
| 158 | request.errback = None |
| 159 | |
| 160 | # Return cached result if request was already seen |
| 161 | if fp in info.downloaded: |
| 162 | await _defer_sleep_async() |
| 163 | cached_result = info.downloaded[fp] |
| 164 | if isinstance(cached_result, Failure): |
| 165 | if eb: |
| 166 | return eb(cached_result) |
| 167 | cached_result.raiseException() |
| 168 | return cached_result |
| 169 | |
| 170 | # Otherwise, wait for result |
| 171 | wad: Deferred[FileInfo] = Deferred() |
| 172 | if eb: |
| 173 | wad.addErrback(eb) |
| 174 | info.waiting[fp].append(wad) |
| 175 | |
| 176 | # Check if request is downloading right now to avoid doing it twice |
| 177 | if fp in info.downloading: |
| 178 | return await maybe_deferred_to_future(wad) |
| 179 | |
| 180 | # Download request checking media_to_download hook output first |
| 181 | info.downloading.add(fp) |
| 182 | await _defer_sleep_async() |
| 183 | result: FileInfo | Failure |
| 184 | try: |
| 185 | file_info: FileInfo | None = await ensure_awaitable( |
| 186 | self.media_to_download(request, info, item=item) |
| 187 | ) |
| 188 | if file_info: |
| 189 | # got a result without downloading |
| 190 | result = file_info |
| 191 | else: |
| 192 | # download the result |
| 193 | result = await self._check_media_to_download(request, info, item=item) |
| 194 | except Exception: |
| 195 | result = Failure() |
| 196 | logger.exception(result) |
| 197 | self._cache_result_and_execute_waiters(result, fp, info) |
| 198 | return await maybe_deferred_to_future(wad) # it must return wad at last |
| 199 | |
| 200 | def _modify_media_request(self, request: Request) -> None: |
| 201 | if self.handle_httpstatus_list: |
no test coverage detected