Iterate over tokens from a possibly-incomplete string of code. This adds two special token types: INCOMPLETE_STRING and IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and represent the two main ways for code to be incomplete.
(s)
| 117 | self.line = line |
| 118 | |
| 119 | def partial_tokens(s): |
| 120 | """Iterate over tokens from a possibly-incomplete string of code. |
| 121 | |
| 122 | This adds two special token types: INCOMPLETE_STRING and |
| 123 | IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and |
| 124 | represent the two main ways for code to be incomplete. |
| 125 | """ |
| 126 | readline = io.StringIO(s).readline |
| 127 | token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '') |
| 128 | try: |
| 129 | for token in tokenize.generate_tokens(readline): |
| 130 | yield token |
| 131 | except tokenize.TokenError as e: |
| 132 | # catch EOF error |
| 133 | lines = s.splitlines(keepends=True) |
| 134 | end = len(lines), len(lines[-1]) |
| 135 | if 'multi-line string' in e.args[0]: |
| 136 | l, c = start = token.end |
| 137 | s = lines[l-1][c:] + ''.join(lines[l:]) |
| 138 | yield IncompleteString(s, start, end, lines[-1]) |
| 139 | elif 'multi-line statement' in e.args[0]: |
| 140 | yield InMultilineStatement(end, lines[-1]) |
| 141 | else: |
| 142 | raise |
| 143 | |
| 144 | def find_next_indent(code): |
| 145 | """Find the number of spaces for the next line of indentation""" |
no test coverage detected