| 69 | _token_pattern = None |
| 70 | |
| 71 | def _tokenize(plural): |
| 72 | global _token_pattern |
| 73 | if _token_pattern is None: |
| 74 | import re |
| 75 | _token_pattern = re.compile(r""" |
| 76 | (?P<WHITESPACES>[ \t]+) | # spaces and horizontal tabs |
| 77 | (?P<NUMBER>[0-9]+\b) | # decimal integer |
| 78 | (?P<NAME>n\b) | # only n is allowed |
| 79 | (?P<PARENTHESIS>[()]) | |
| 80 | (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >, |
| 81 | # <=, >=, ==, !=, &&, ||, |
| 82 | # ? : |
| 83 | # unary and bitwise ops |
| 84 | # not allowed |
| 85 | (?P<INVALID>\w+|.) # invalid token |
| 86 | """, re.VERBOSE|re.DOTALL) |
| 87 | |
| 88 | for mo in _token_pattern.finditer(plural): |
| 89 | kind = mo.lastgroup |
| 90 | if kind == 'WHITESPACES': |
| 91 | continue |
| 92 | value = mo.group(kind) |
| 93 | if kind == 'INVALID': |
| 94 | raise ValueError('invalid token in plural form: %s' % value) |
| 95 | yield value |
| 96 | yield '' |
| 97 | |
| 98 | |
| 99 | def _error(value): |