Method __init__

llms/speculative_decoding/decoder.py:9–15

(self, model_name: str)

class Tokenizer:
    def __init__(self, model_name: str):
        self._tokenizer = transformers.AutoTokenizer.from_pretrained(
            model_name,
            legacy=False,
            model_max_length=512,
        )
        self._decoder_start_id = 0

    @property
    def eos_id(self) -> int: