Create a new priority queue instance, with disk storage
(self)
| 456 | ) |
| 457 | |
| 458 | def _dq(self) -> ScrapyPriorityQueue: |
| 459 | """Create a new priority queue instance, with disk storage""" |
| 460 | assert self.crawler |
| 461 | assert self.dqdir |
| 462 | assert self.pqclass |
| 463 | state = self._read_dqs_state(self.dqdir) |
| 464 | q = build_from_crawler( |
| 465 | self.pqclass, |
| 466 | self.crawler, |
| 467 | downstream_queue_cls=self.dqclass, |
| 468 | key=self.dqdir, |
| 469 | startprios=state, |
| 470 | start_queue_cls=self._sdqclass, |
| 471 | ) |
| 472 | if q: |
| 473 | logger.info( |
| 474 | "Resuming crawl (%(queuesize)d requests scheduled)", |
| 475 | {"queuesize": len(q)}, |
| 476 | extra={"spider": self.spider}, |
| 477 | ) |
| 478 | return q |
| 479 | |
| 480 | def _dqdir(self, jobdir: str | None) -> str | None: |
| 481 | """Return a folder name to keep disk queue state at""" |
no test coverage detected