Function tokenizer

tests/model_executor/test_ernie_tokenizer.py:48–53

Create a tokenizer instance with mocked dependencies

(mock_sp_model, vocab_file)

@pytest.fixture
def tokenizer(mock_sp_model, vocab_file):
    """Create a tokenizer instance with mocked dependencies"""
    with patch.object(SentencePieceProcessor, "Load", return_value=None):
        with patch("sentencepiece.SentencePieceProcessor") as mock_spm:
            mock_spm.return_value = mock_sp_model
            return Ernie4_5Tokenizer(vocab_file=vocab_file)


# ===== Initialization Tests (Parametrized) =====