Method test_enumerate_tokenization

tests/test_string_util.py:10–15

(self)

class TestStringUtility(unittest.TestCase):
    def test_enumerate_tokenization(self):
        text = '商品和服务'
        toks = possible_tokenization(text)
        assert len(set(toks)) == 2 ** (len(text) - 1)
        for each in toks:
            assert ''.join(each) == text


if __name__ == '__main__':