Function export_vocabulary

fastdeploy/entrypoints/cli/tokenizer.py:155–184

导出词表到文件

(tokenizer, file_path: str)

def export_vocabulary(tokenizer, file_path: str) -> None:
    """导出词表到文件"""
    try:
        vocab = get_vocab_dict(tokenizer)
        if not vocab:
            print("Warning: Could not retrieve vocabulary from tokenizer")
            return

        path = Path(file_path)
        path.parent.mkdir(parents=True, exist_ok=True)

        # 根据文件扩展名选择格式
        if path.suffix.lower() == ".json":
            with open(path, "w", encoding="utf-8") as f:
                json.dump(vocab, f, ensure_ascii=False, indent=2)
        else:
            # 默认格式：每行一个token
            with open(path, "w", encoding="utf-8") as f:
                for token, token_id in sorted(vocab.items(), key=lambda x: x[1]):
                    # 处理不可打印字符
                    try:
                        f.write(f"{token_id}\t{repr(token)}\n")
                    except:
                        f.write(f"{token_id}\t<unprintable>\n")

        print(f"Vocabulary exported to: {file_path}")
        print(f"Total tokens: {len(vocab)}")

    except Exception as e:
        print(f"Error exporting vocabulary: {e}")


def main(args: argparse.Namespace) -> None: