Caching wrapper for the tokenize module. This is pretty tied to Python's syntax.
| 16 | |
| 17 | |
| 18 | class Tokenizer: |
| 19 | """Caching wrapper for the tokenize module. |
| 20 | |
| 21 | This is pretty tied to Python's syntax. |
| 22 | """ |
| 23 | |
| 24 | _tokens: list[tokenize.TokenInfo] |
| 25 | |
| 26 | def __init__( |
| 27 | self, tokengen: Iterator[tokenize.TokenInfo], *, path: str = "", verbose: bool = False |
| 28 | ): |
| 29 | self._tokengen = tokengen |
| 30 | self._tokens = [] |
| 31 | self._index = 0 |
| 32 | self._verbose = verbose |
| 33 | self._lines: dict[int, str] = {} |
| 34 | self._path = path |
| 35 | if verbose: |
| 36 | self.report(False, False) |
| 37 | |
| 38 | def getnext(self) -> tokenize.TokenInfo: |
| 39 | """Return the next token and updates the index.""" |
| 40 | cached = not self._index == len(self._tokens) |
| 41 | tok = self.peek() |
| 42 | self._index += 1 |
| 43 | if self._verbose: |
| 44 | self.report(cached, False) |
| 45 | return tok |
| 46 | |
| 47 | def peek(self) -> tokenize.TokenInfo: |
| 48 | """Return the next token *without* updating the index.""" |
| 49 | while self._index == len(self._tokens): |
| 50 | tok = next(self._tokengen) |
| 51 | if tok.type in (tokenize.NL, tokenize.COMMENT): |
| 52 | continue |
| 53 | if tok.type == token.ERRORTOKEN and tok.string.isspace(): |
| 54 | continue |
| 55 | if ( |
| 56 | tok.type == token.NEWLINE |
| 57 | and self._tokens |
| 58 | and self._tokens[-1].type == token.NEWLINE |
| 59 | ): |
| 60 | continue |
| 61 | self._tokens.append(tok) |
| 62 | if not self._path: |
| 63 | self._lines[tok.start[0]] = tok.line |
| 64 | return self._tokens[self._index] |
| 65 | |
| 66 | def diagnose(self) -> tokenize.TokenInfo: |
| 67 | if not self._tokens: |
| 68 | self.getnext() |
| 69 | return self._tokens[-1] |
| 70 | |
| 71 | def get_last_non_whitespace_token(self) -> tokenize.TokenInfo: |
| 72 | for tok in reversed(self._tokens[: self._index]): |
| 73 | if tok.type != tokenize.ENDMARKER and ( |
| 74 | tok.type < tokenize.NEWLINE or tok.type > tokenize.DEDENT |
| 75 | ): |
no outgoing calls
searching dependent graphs…