Function new_encode

tpu_mtj_backend.py:1157–1160

(old_encode)

    if vars.model == "TPUMeshTransformerGPTNeoX":
        tokenizer = Tokenizer.from_file(os.path.join(path, "20B_tokenizer.json"))
        def new_encode(old_encode):
            def encode(s, *args, **kwargs):
                return old_encode(s).ids
            return encode
        tokenizer.encode = new_encode(tokenizer.encode)
        tokenizer._koboldai_header = []
    elif not hf_checkpoint: