Parse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines.
(self, lines)
| 95 | self.entries.append(entry) |
| 96 | |
| 97 | def parse(self, lines): |
| 98 | """Parse the input lines from a robots.txt file. |
| 99 | |
| 100 | We allow that a user-agent: line is not preceded by |
| 101 | one or more blank lines. |
| 102 | """ |
| 103 | # states: |
| 104 | # 0: start state |
| 105 | # 1: saw user-agent line |
| 106 | # 2: saw an allow or disallow line |
| 107 | state = 0 |
| 108 | entry = Entry() |
| 109 | |
| 110 | self.modified() |
| 111 | for line in lines: |
| 112 | if not line: |
| 113 | if state == 1: |
| 114 | entry = Entry() |
| 115 | state = 0 |
| 116 | elif state == 2: |
| 117 | self._add_entry(entry) |
| 118 | entry = Entry() |
| 119 | state = 0 |
| 120 | # remove optional comment and strip line |
| 121 | i = line.find('#') |
| 122 | if i >= 0: |
| 123 | line = line[:i] |
| 124 | line = line.strip() |
| 125 | if not line: |
| 126 | continue |
| 127 | line = line.split(':', 1) |
| 128 | if len(line) == 2: |
| 129 | line[0] = line[0].strip().lower() |
| 130 | line[1] = line[1].strip() |
| 131 | if line[0] == "user-agent": |
| 132 | if state == 2: |
| 133 | self._add_entry(entry) |
| 134 | entry = Entry() |
| 135 | entry.useragents.append(line[1]) |
| 136 | state = 1 |
| 137 | elif line[0] == "disallow": |
| 138 | if state != 0: |
| 139 | entry.rulelines.append(RuleLine(line[1], False)) |
| 140 | state = 2 |
| 141 | elif line[0] == "allow": |
| 142 | if state != 0: |
| 143 | entry.rulelines.append(RuleLine(line[1], True)) |
| 144 | state = 2 |
| 145 | elif line[0] == "crawl-delay": |
| 146 | if state != 0: |
| 147 | # before trying to convert to int we need to make |
| 148 | # sure that robots.txt has valid syntax otherwise |
| 149 | # it will crash |
| 150 | if line[1].strip().isdigit(): |
| 151 | entry.delay = int(line[1]) |
| 152 | state = 2 |
| 153 | elif line[0] == "request-rate": |
| 154 | if state != 0: |