Function tokenize

src/lib/libc_preprocessor.js:88–113

* @param {string} exprString * @param {(number|boolean)=} keepWhitespace Optional, can be omitted. Defaults to false.

(exprString, keepWhitespace)

     * @param {(number|boolean)=} keepWhitespace Optional, can be omitted. Defaults to false.
    */
    function tokenize(exprString, keepWhitespace) {
      var out = [], len = exprString.length;
      for (var i = 0; i <= len; ++i) {
        var kind = classifyChar(exprString, i);
        if (kind == 2/*0-9*/ || kind == 3/*a-z*/) { // a character or a number
          for (var j = i+1; j <= len; ++j) {
            var kind2 = classifyChar(exprString, j);
            if (kind2 != kind && (kind2 != 2/*0-9*/ || kind != 3/*a-z*/)) { // parse number sequence "423410", and identifier sequence "FOO32BAR"
              out.push(exprString.substring(i, j));
              i = j-1;
              break;
            }
          }
        } else if (kind == 1/*operator symbol*/) {
          // Lookahead for two-character operators.
          var op2 = exprString.slice(i, i + 2);
          if (['<=', '>=', '==', '!=', '&&', '||'].includes(op2)) {
            out.push(op2);
            ++i;
          } else {
            out.push(exprString[i]);
          }
        }
      }
      return out;
    }

    // Expands preprocessing macros on substring str[lineStart...lineEnd]
    /**