Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is g
(self)
| 558 | ) |
| 559 | |
| 560 | def _spider_idle(self) -> None: |
| 561 | """ |
| 562 | Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. |
| 563 | It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider |
| 564 | exception, the spider is not closed until the next loop and this function is guaranteed to be called |
| 565 | (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. |
| 566 | """ |
| 567 | assert self.spider is not None # typing |
| 568 | expected_ex = (DontCloseSpider, CloseSpider) |
| 569 | res = self.signals.send_catch_log( |
| 570 | signals.spider_idle, spider=self.spider, dont_log=expected_ex |
| 571 | ) |
| 572 | detected_ex = { |
| 573 | ex: x.value |
| 574 | for _, x in res |
| 575 | for ex in expected_ex |
| 576 | if isinstance(x, Failure) and isinstance(x.value, ex) |
| 577 | } |
| 578 | if DontCloseSpider in detected_ex: |
| 579 | return |
| 580 | if self.spider_is_idle(): |
| 581 | ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) |
| 582 | assert isinstance(ex, CloseSpider) # typing |
| 583 | _schedule_coro(self.close_spider_async(reason=ex.reason)) |
| 584 | |
| 585 | def close_spider( |
| 586 | self, spider: Spider, reason: str = "cancelled" |
no test coverage detected