获取tokenizer的元信息
(tokenizer)
| 85 | |
| 86 | |
| 87 | def get_tokenizer_info(tokenizer) -> dict: |
| 88 | """获取tokenizer的元信息""" |
| 89 | info = {} |
| 90 | |
| 91 | try: |
| 92 | # 基本属性 |
| 93 | info["vocab_size"] = get_vocab_size(tokenizer) |
| 94 | |
| 95 | # 模型类型和路径 |
| 96 | if hasattr(tokenizer, "name_or_path"): |
| 97 | info["model_name"] = tokenizer.name_or_path |
| 98 | |
| 99 | # tokenizer类型 |
| 100 | info["tokenizer_type"] = type(tokenizer).__name__ |
| 101 | |
| 102 | # 特殊符号 |
| 103 | special_tokens = {} |
| 104 | for attr in ["bos_token", "eos_token", "unk_token", "sep_token", "pad_token", "cls_token", "mask_token"]: |
| 105 | if hasattr(tokenizer, attr): |
| 106 | token = getattr(tokenizer, attr) |
| 107 | if token: |
| 108 | special_tokens[attr] = token |
| 109 | info["special_tokens"] = special_tokens |
| 110 | |
| 111 | # 特殊token IDs |
| 112 | special_token_ids = {} |
| 113 | for attr in [ |
| 114 | "bos_token_id", |
| 115 | "eos_token_id", |
| 116 | "unk_token_id", |
| 117 | "sep_token_id", |
| 118 | "pad_token_id", |
| 119 | "cls_token_id", |
| 120 | "mask_token_id", |
| 121 | ]: |
| 122 | if hasattr(tokenizer, attr): |
| 123 | token_id = getattr(tokenizer, attr) |
| 124 | if token_id is not None: |
| 125 | special_token_ids[attr] = token_id |
| 126 | info["special_token_ids"] = special_token_ids |
| 127 | |
| 128 | # 模型最大长度 |
| 129 | if hasattr(tokenizer, "model_max_length"): |
| 130 | info["model_max_length"] = tokenizer.model_max_length |
| 131 | |
| 132 | except Exception as e: |
| 133 | info["error"] = f"Failed to get tokenizer info: {e}" |
| 134 | |
| 135 | return info |
| 136 | |
| 137 | |
| 138 | def get_vocab_dict(tokenizer) -> dict: |