using the parsed robots.txt decide if useragent can fetch url
(self, useragent, url)
| 168 | self._add_entry(entry) |
| 169 | |
| 170 | def can_fetch(self, useragent, url): |
| 171 | """using the parsed robots.txt decide if useragent can fetch url""" |
| 172 | if self.disallow_all: |
| 173 | return False |
| 174 | if self.allow_all: |
| 175 | return True |
| 176 | # Until the robots.txt file has been read or found not |
| 177 | # to exist, we must assume that no url is allowable. |
| 178 | # This prevents false positives when a user erroneously |
| 179 | # calls can_fetch() before calling read(). |
| 180 | if not self.last_checked: |
| 181 | return False |
| 182 | # search for given user agent matches |
| 183 | # the first match counts |
| 184 | # TODO: The private API is used in order to preserve an empty query. |
| 185 | # This is temporary until the public API starts supporting this feature. |
| 186 | parsed_url = urllib.parse._urlsplit(url, '') |
| 187 | url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) |
| 188 | url = normalize_path(url) |
| 189 | if not url: |
| 190 | url = "/" |
| 191 | for entry in self.entries: |
| 192 | if entry.applies_to(useragent): |
| 193 | return entry.allowance(url) |
| 194 | # try the default entry last |
| 195 | if self.default_entry: |
| 196 | return self.default_entry.allowance(url) |
| 197 | # agent not found ==> access granted |
| 198 | return True |
| 199 | |
| 200 | def crawl_delay(self, useragent): |
| 201 | if not self.mtime(): |