(self, string)
| 229 | |
| 230 | class Tokenizer: |
| 231 | def __init__(self, string): |
| 232 | self.istext = isinstance(string, str) |
| 233 | self.string = string |
| 234 | if not self.istext: |
| 235 | string = str(string, 'latin1') |
| 236 | self.decoded_string = string |
| 237 | self.index = 0 |
| 238 | self.next = None |
| 239 | self.__next() |
| 240 | def __next(self): |
| 241 | index = self.index |
| 242 | try: |