| 548 | // Tokenize into word runs, whitespace runs, and single punctuation chars — |
| 549 | // matches the Rust tokenize() which mirrors diffWordsWithSpace's splitting. |
| 550 | function tokenize(text: string): string[] { |
| 551 | const tokens: string[] = [] |
| 552 | let i = 0 |
| 553 | while (i < text.length) { |
| 554 | const ch = text[i]! |
| 555 | if (/[\p{L}\p{N}_]/u.test(ch)) { |
| 556 | let j = i + 1 |
| 557 | while (j < text.length && /[\p{L}\p{N}_]/u.test(text[j]!)) j++ |
| 558 | tokens.push(text.slice(i, j)) |
| 559 | i = j |
| 560 | } else if (/\s/.test(ch)) { |
| 561 | let j = i + 1 |
| 562 | while (j < text.length && /\s/.test(text[j]!)) j++ |
| 563 | tokens.push(text.slice(i, j)) |
| 564 | i = j |
| 565 | } else { |
| 566 | // advance one codepoint (handle surrogate pairs) |
| 567 | const cp = text.codePointAt(i)! |
| 568 | const len = cp > 0xffff ? 2 : 1 |
| 569 | tokens.push(text.slice(i, i + len)) |
| 570 | i += len |
| 571 | } |
| 572 | } |
| 573 | return tokens |
| 574 | } |
| 575 | |
| 576 | function findAdjacentPairs(markers: Marker[]): [number, number][] { |
| 577 | const pairs: [number, number][] = [] |