| 253 | |
| 254 | |
| 255 | def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions: |
| 256 | all_tokens = {} |
| 257 | exact_tokens = {} |
| 258 | non_exact_tokens = set() |
| 259 | numbers = itertools.count(0) |
| 260 | |
| 261 | for line in tokens: |
| 262 | line = line.strip() |
| 263 | |
| 264 | if not line or line.startswith("#"): |
| 265 | continue |
| 266 | |
| 267 | pieces = line.split() |
| 268 | index = next(numbers) |
| 269 | |
| 270 | if len(pieces) == 1: |
| 271 | (token,) = pieces |
| 272 | non_exact_tokens.add(token) |
| 273 | all_tokens[index] = token |
| 274 | elif len(pieces) == 2: |
| 275 | token, op = pieces |
| 276 | exact_tokens[op.strip("'")] = index |
| 277 | all_tokens[index] = token |
| 278 | else: |
| 279 | raise ValueError(f"Unexpected line found in Tokens file: {line}") |
| 280 | |
| 281 | return all_tokens, exact_tokens, non_exact_tokens |
| 282 | |
| 283 | |
| 284 | def build_c_generator( |