()
| 67 | |
| 68 | |
| 69 | def test_basic_encode(): |
| 70 | enc = tiktoken.get_encoding("r50k_base") |
| 71 | assert enc.encode("hello world") == [31373, 995] |
| 72 | |
| 73 | enc = tiktoken.get_encoding("p50k_base") |
| 74 | assert enc.encode("hello world") == [31373, 995] |
| 75 | |
| 76 | enc = tiktoken.get_encoding("cl100k_base") |
| 77 | assert enc.encode("hello world") == [15339, 1917] |
| 78 | assert enc.encode(" \x850") == [220, 126, 227, 15] |
| 79 | |
| 80 | |
| 81 | def test_encode_empty(): |
nothing calls this directly
no test coverage detected
searching dependent graphs…