Reads the robots.txt URL and feeds it to the parser.
(self)
| 72 | self.host, self.path = urllib.parse.urlsplit(url)[1:3] |
| 73 | |
| 74 | def read(self): |
| 75 | """Reads the robots.txt URL and feeds it to the parser.""" |
| 76 | try: |
| 77 | f = urllib.request.urlopen(self.url) |
| 78 | except urllib.error.HTTPError as err: |
| 79 | if err.code in (401, 403): |
| 80 | self.disallow_all = True |
| 81 | elif err.code >= 400 and err.code < 500: |
| 82 | self.allow_all = True |
| 83 | err.close() |
| 84 | else: |
| 85 | raw = f.read() |
| 86 | self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) |
| 87 | |
| 88 | def _add_entry(self, entry): |
| 89 | if "*" in entry.useragents: |