| 25 | // primary → NAME | STRING | BOOLEAN |
| 26 | // | "(" expression ")" ; |
| 27 | function tokenize(code) { |
| 28 | const tokens = []; |
| 29 | let i = 0; |
| 30 | while (i < code.length) { |
| 31 | let char = code[i]; |
| 32 | // Double quoted strings |
| 33 | if (char === '"') { |
| 34 | let string = ''; |
| 35 | i++; |
| 36 | do { |
| 37 | if (i > code.length) { |
| 38 | throw Error('Missing a closing quote'); |
| 39 | } |
| 40 | char = code[i++]; |
| 41 | if (char === '"') { |
| 42 | break; |
| 43 | } |
| 44 | string += char; |
| 45 | } while (true); |
| 46 | tokens.push({type: 'string', value: string}); |
| 47 | continue; |
| 48 | } |
| 49 | |
| 50 | // Single quoted strings |
| 51 | if (char === "'") { |
| 52 | let string = ''; |
| 53 | i++; |
| 54 | do { |
| 55 | if (i > code.length) { |
| 56 | throw Error('Missing a closing quote'); |
| 57 | } |
| 58 | char = code[i++]; |
| 59 | if (char === "'") { |
| 60 | break; |
| 61 | } |
| 62 | string += char; |
| 63 | } while (true); |
| 64 | tokens.push({type: 'string', value: string}); |
| 65 | continue; |
| 66 | } |
| 67 | |
| 68 | // Whitespace |
| 69 | if (/\s/.test(char)) { |
| 70 | if (char === '\n') { |
| 71 | return tokens; |
| 72 | } |
| 73 | i++; |
| 74 | continue; |
| 75 | } |
| 76 | |
| 77 | const next3 = code.slice(i, i + 3); |
| 78 | if (next3 === '===') { |
| 79 | tokens.push({type: '=='}); |
| 80 | i += 3; |
| 81 | continue; |
| 82 | } |
| 83 | if (next3 === '!==') { |
| 84 | tokens.push({type: '!='}); |