| 283 | |
| 284 | # Line-eater for tokenize. |
| 285 | def tokeneater(self, type, token, slinecol, end, line, |
| 286 | INDENT=tokenize.INDENT, |
| 287 | DEDENT=tokenize.DEDENT, |
| 288 | NEWLINE=tokenize.NEWLINE, |
| 289 | COMMENT=tokenize.COMMENT, |
| 290 | NL=tokenize.NL): |
| 291 | |
| 292 | if type == NEWLINE: |
| 293 | # A program statement, or ENDMARKER, will eventually follow, |
| 294 | # after some (possibly empty) run of tokens of the form |
| 295 | # (NL | COMMENT)* (INDENT | DEDENT+)? |
| 296 | self.find_stmt = 1 |
| 297 | |
| 298 | elif type == INDENT: |
| 299 | self.find_stmt = 1 |
| 300 | self.level += 1 |
| 301 | |
| 302 | elif type == DEDENT: |
| 303 | self.find_stmt = 1 |
| 304 | self.level -= 1 |
| 305 | |
| 306 | elif type == COMMENT: |
| 307 | if self.find_stmt: |
| 308 | self.stats.append((slinecol[0], -1)) |
| 309 | # but we're still looking for a new stmt, so leave |
| 310 | # find_stmt alone |
| 311 | |
| 312 | elif type == NL: |
| 313 | pass |
| 314 | |
| 315 | elif self.find_stmt: |
| 316 | # This is the first "real token" following a NEWLINE, so it |
| 317 | # must be the first token of the next program statement, or an |
| 318 | # ENDMARKER. |
| 319 | self.find_stmt = 0 |
| 320 | if line: # not endmarker |
| 321 | self.stats.append((slinecol[0], self.level)) |
| 322 | |
| 323 | |
| 324 | # Count number of leading blanks. |