Method can_fetch

scrapling/spiders/robotstxt.py:43–50

Check if a URL can be fetched according to the domain's robots.txt. :param url: The full URL to check :param sid: Session ID for fetching robots.txt if not yet cached

(self, url: str, sid: str)

        return parser

    async def can_fetch(self, url: str, sid: str) -> bool:
        """Check if a URL can be fetched according to the domain's robots.txt.

        :param url: The full URL to check
        :param sid: Session ID for fetching robots.txt if not yet cached
        """
        parser = await self._get_parser(url, sid)
        return parser.can_fetch(url, "*")

    async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
        """Return both crawl-delay and request-rate in a single parser lookup.