(args=None)
| 506 | return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True) |
| 507 | |
| 508 | def _main(args=None): |
| 509 | import argparse |
| 510 | |
| 511 | # Helper error handling routines |
| 512 | def perror(message): |
| 513 | sys.stderr.write(message) |
| 514 | sys.stderr.write('\n') |
| 515 | |
| 516 | def error(message, filename=None, location=None): |
| 517 | if location: |
| 518 | args = (filename,) + location + (message,) |
| 519 | perror("%s:%d:%d: error: %s" % args) |
| 520 | elif filename: |
| 521 | perror("%s: error: %s" % (filename, message)) |
| 522 | else: |
| 523 | perror("error: %s" % message) |
| 524 | sys.exit(1) |
| 525 | |
| 526 | # Parse the arguments and options |
| 527 | parser = argparse.ArgumentParser(color=True) |
| 528 | parser.add_argument(dest='filename', nargs='?', |
| 529 | metavar='filename.py', |
| 530 | help='the file to tokenize; defaults to stdin') |
| 531 | parser.add_argument('-e', '--exact', dest='exact', action='store_true', |
| 532 | help='display token names using the exact type') |
| 533 | args = parser.parse_args(args) |
| 534 | |
| 535 | try: |
| 536 | # Tokenize the input |
| 537 | if args.filename: |
| 538 | filename = args.filename |
| 539 | with _builtin_open(filename, 'rb') as f: |
| 540 | tokens = list(tokenize(f.readline)) |
| 541 | else: |
| 542 | filename = "<stdin>" |
| 543 | tokens = _generate_tokens_from_c_tokenizer( |
| 544 | sys.stdin.readline, extra_tokens=True) |
| 545 | |
| 546 | |
| 547 | # Output the tokenization |
| 548 | for token in tokens: |
| 549 | token_type = token.type |
| 550 | if args.exact: |
| 551 | token_type = token.exact_type |
| 552 | token_range = "%d,%d-%d,%d:" % (token.start + token.end) |
| 553 | print("%-20s%-15s%-15r" % |
| 554 | (token_range, tok_name[token_type], token.string)) |
| 555 | except IndentationError as err: |
| 556 | line, column = err.args[1][1:3] |
| 557 | error(err.args[0], filename, (line, column)) |
| 558 | except TokenError as err: |
| 559 | line, column = err.args[1] |
| 560 | error(err.args[0], filename, (line, column)) |
| 561 | except SyntaxError as err: |
| 562 | error(err, filename) |
| 563 | except OSError as err: |
| 564 | error(err) |
| 565 | except KeyboardInterrupt: |
no test coverage detected
searching dependent graphs…