Function possible_tokenization

hanlp/utils/string_util.py:99–117

Enumerate all possible tokenizations of a text. Args: text: A text. Returns: All possible tokenizations.

(text: str)

def possible_tokenization(text: str) -> List[Tuple[str]]:
    """Enumerate all possible tokenizations of a text.

    Args:
        text: A text.

    Returns: All possible tokenizations.

    """
    states = [((), ())]
    for c in text:
        new_states = []
        for t, b in states:
            # to split
            new_states.append((t + (''.join(b + (c,)),), ()))
            # not to split
            new_states.append((t, b + (c,)))
        states = new_states
    return [t for t, b in states if not b]