| 225 | failure.raiseException() |
| 226 | |
| 227 | def _cache_result_and_execute_waiters( |
| 228 | self, result: FileInfo | Failure, fp: bytes, info: SpiderInfo |
| 229 | ) -> None: |
| 230 | if isinstance(result, Failure): |
| 231 | # minimize cached information for failure |
| 232 | result.cleanFailure() |
| 233 | result.frames.clear() |
| 234 | if TWISTED_FAILURE_HAS_STACK: |
| 235 | result.stack.clear() |
| 236 | # This code fixes a memory leak by avoiding to keep references to |
| 237 | # the Request and Response objects on the Media Pipeline cache. |
| 238 | # |
| 239 | # What happens when the media_downloaded callback raises an |
| 240 | # exception, for example a FileException('download-error') when |
| 241 | # the Response status code is not 200 OK, is that the original |
| 242 | # StopIteration exception (which in turn contains the failed |
| 243 | # Response and by extension, the original Request) gets encapsulated |
| 244 | # within the FileException context. |
| 245 | # |
| 246 | # Originally, Scrapy was using twisted.internet.defer.returnValue |
| 247 | # inside functions decorated with twisted.internet.defer.inlineCallbacks, |
| 248 | # encapsulating the returned Response in a _DefGen_Return exception |
| 249 | # instead of a StopIteration. |
| 250 | # |
| 251 | # To avoid keeping references to the Response and therefore Request |
| 252 | # objects on the Media Pipeline cache, we should wipe the context of |
| 253 | # the encapsulated exception when it is a StopIteration instance |
| 254 | context = getattr(result.value, "__context__", None) |
| 255 | if isinstance(context, StopIteration): |
| 256 | assert result.value is not None |
| 257 | result.value.__context__ = None |
| 258 | |
| 259 | info.downloading.remove(fp) |
| 260 | info.downloaded[fp] = result # cache result |
| 261 | for wad in info.waiting.pop(fp): |
| 262 | if isinstance(result, Failure): |
| 263 | call_later(_DEFER_DELAY, wad.errback, result) |
| 264 | else: |
| 265 | call_later(_DEFER_DELAY, wad.callback, result) |
| 266 | |
| 267 | # Overridable Interface |
| 268 | @abstractmethod |