(self, request: Request, spider: Spider)
| 65 | raise IgnoreRequest |
| 66 | |
| 67 | def should_follow(self, request: Request, spider: Spider) -> bool: |
| 68 | regex = self.host_regex |
| 69 | # hostname can be None for wrong urls (like javascript links) |
| 70 | host = urlparse_cached(request).hostname or "" |
| 71 | return bool(regex.search(host)) |
| 72 | |
| 73 | def get_host_regex(self, spider: Spider) -> re.Pattern[str]: |
| 74 | """Override this method to implement a different offsite policy""" |
no test coverage detected