Tokenize a series of lines and group tokens by line. The tokens for a multiline Python string or expression are grouped as one line. All lines except the last lines should keep their line ending ('\\n', '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)` for
(lines:List[str])
| 466 | return lines_before + [new_line] + lines_after |
| 467 | |
| 468 | def make_tokens_by_line(lines:List[str]): |
| 469 | """Tokenize a series of lines and group tokens by line. |
| 470 | |
| 471 | The tokens for a multiline Python string or expression are grouped as one |
| 472 | line. All lines except the last lines should keep their line ending ('\\n', |
| 473 | '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)` |
| 474 | for example when passing block of text to this function. |
| 475 | |
| 476 | """ |
| 477 | # NL tokens are used inside multiline expressions, but also after blank |
| 478 | # lines or comments. This is intentional - see https://bugs.python.org/issue17061 |
| 479 | # We want to group the former case together but split the latter, so we |
| 480 | # track parentheses level, similar to the internals of tokenize. |
| 481 | NEWLINE, NL = tokenize.NEWLINE, tokenize.NL |
| 482 | tokens_by_line = [[]] |
| 483 | if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')): |
| 484 | warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified") |
| 485 | parenlev = 0 |
| 486 | try: |
| 487 | for token in tokenize.generate_tokens(iter(lines).__next__): |
| 488 | tokens_by_line[-1].append(token) |
| 489 | if (token.type == NEWLINE) \ |
| 490 | or ((token.type == NL) and (parenlev <= 0)): |
| 491 | tokens_by_line.append([]) |
| 492 | elif token.string in {'(', '[', '{'}: |
| 493 | parenlev += 1 |
| 494 | elif token.string in {')', ']', '}'}: |
| 495 | if parenlev > 0: |
| 496 | parenlev -= 1 |
| 497 | except tokenize.TokenError: |
| 498 | # Input ended in a multiline string or expression. That's OK for us. |
| 499 | pass |
| 500 | |
| 501 | |
| 502 | if not tokens_by_line[-1]: |
| 503 | tokens_by_line.pop() |
| 504 | |
| 505 | |
| 506 | return tokens_by_line |
| 507 | |
| 508 | def show_linewise_tokens(s: str): |
| 509 | """For investigation and debugging""" |