Function make_tokens_by_line

IPython/core/inputtransformer2.py:598–643

Tokenize a series of lines and group tokens by line. The tokens for a multiline Python string or expression are grouped as one line. All lines except the last lines should keep their line ending ('\\n', '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)` for

(lines:List[str])

        return lines_before + [new_line] + lines_after

def make_tokens_by_line(lines:List[str]):
    """Tokenize a series of lines and group tokens by line.

    The tokens for a multiline Python string or expression are grouped as one
    line. All lines except the last lines should keep their line ending ('\\n',
    '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
    for example when passing block of text to this function.

    """
    # NL tokens are used inside multiline expressions, but also after blank
    # lines or comments. This is intentional - see https://bugs.python.org/issue17061
    # We want to group the former case together but split the latter, so we
    # track parentheses level, similar to the internals of tokenize.

    #   reexported from token on 3.7+
    NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
    tokens_by_line: List[List[Any]] = [[]]
    if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
        warnings.warn(
            "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
            stacklevel=2,
        )
    parenlev = 0
    try:
        for token in tokenutil.generate_tokens_catch_errors(
            iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
        ):
            tokens_by_line[-1].append(token)
            if (token.type == NEWLINE) \
                    or ((token.type == NL) and (parenlev <= 0)):
                tokens_by_line.append([])
            elif token.string in {'(', '[', '{'}:
                parenlev += 1
            elif token.string in {')', ']', '}'}:
                if parenlev > 0:
                    parenlev -= 1
    except tokenize.TokenError:
        # Input ended in a multiline string or expression. That's OK for us.
        pass


    if not tokens_by_line[-1]:
        tokens_by_line.pop()


    return tokens_by_line


def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):