Function bpe

python/mlx/data/tokenizer_helpers.py:168–186

(tokenmap, token, max_rank)

    merges = BPEMerges()

    def bpe(tokenmap, token, max_rank):
        parts = list(token)
        while True:
            min_idx = None
            min_rank = None
            for i, pair in enumerate(zip(parts[:-1], parts[1:])):
                rank = tokenmap.get((pair[0] + pair[1]).encode())
                if rank is not None and (min_rank is None or rank < min_rank):
                    min_idx = i
                    min_rank = rank
            if min_rank is None or (max_rank is not None and min_rank >= max_rank):
                break
            assert min_idx is not None
            parts = (
                parts[:min_idx]
                + [parts[min_idx] + parts[min_idx + 1]]
                + parts[min_idx + 2 :]
            )
        return parts

    for t in merged:
        left, right = bpe(tokenmap, t.decode(), tokenmap[t])