Function shorttok

Tools/peg_generator/pegen/tokenizer.py:10–15

(tok: tokenize.TokenInfo)

def shorttok(tok: tokenize.TokenInfo) -> str:
    formatted = (
        f"{tok.start[0]}.{tok.start[1]}: "
        f"{token.tok_name[tok.type]}:{tok.string!r}"
    )
    return f"{formatted:<25.25}"


class Tokenizer: