Function generate_tokens

IPython/utils/tokenutil.py:15–22

wrap generate_tokens to catch EOF errors

(readline)

Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])

def generate_tokens(readline):
    """wrap generate_tokens to catch EOF errors"""
    try:
        for token in tokenize.generate_tokens(readline):
            yield token
    except tokenize.TokenError:
        # catch EOF error
        return

def line_at_cursor(cell, cursor_pos=0):
    """Return the line in a cell at a given cursor position