* For testing purposes. * This method allows us to run and assert the proper working of the tokenizer.
()
| 130 | * This method allows us to run and assert the proper working of the tokenizer. |
| 131 | */ |
| 132 | tokenize(): InputToken[] { |
| 133 | const tokens: InputToken[] = []; |
| 134 | let inputToken: InputToken; |
| 135 | do { |
| 136 | inputToken = this.consumeAToken(); |
| 137 | tokens.push(inputToken); |
| 138 | } while (inputToken); |
| 139 | |
| 140 | return tokens; |
| 141 | } |
| 142 | |
| 143 | /** |
| 144 | * 4.3.1. Consume a token |
no test coverage detected