(
self,
response: Response,
exception: Exception,
start_index: int = 0,
)
| 114 | recover_to.extend(exception_result) |
| 115 | |
| 116 | def _process_spider_exception( |
| 117 | self, |
| 118 | response: Response, |
| 119 | exception: Exception, |
| 120 | start_index: int = 0, |
| 121 | ) -> MutableAsyncChain[_T]: |
| 122 | # don't handle _InvalidOutput exception |
| 123 | if isinstance(exception, _InvalidOutput): |
| 124 | raise exception |
| 125 | method_list = islice( |
| 126 | self.methods["process_spider_exception"], start_index, None |
| 127 | ) |
| 128 | for method_index, method in enumerate(method_list, start=start_index): |
| 129 | if method is None: |
| 130 | continue |
| 131 | if method in self._mw_methods_requiring_spider: |
| 132 | result = method( |
| 133 | response=response, exception=exception, spider=self._spider |
| 134 | ) |
| 135 | else: |
| 136 | result = method(response=response, exception=exception) |
| 137 | if isinstance(result, (Iterable, AsyncIterator)): |
| 138 | # stop exception handling by handing control over to the |
| 139 | # process_spider_output chain if an iterable has been returned |
| 140 | if isinstance(result, Iterable): |
| 141 | result = as_async_generator(result) |
| 142 | return self._process_spider_output(response, result, method_index + 1) |
| 143 | if result is None: |
| 144 | continue |
| 145 | msg = ( |
| 146 | f"{global_object_name(method)} must return None " |
| 147 | f"or an iterable, got {type(result)}" |
| 148 | ) |
| 149 | raise _InvalidOutput(msg) |
| 150 | raise exception |
| 151 | |
| 152 | def _process_spider_output( |
| 153 | self, |
no test coverage detected