Function build_processor

utils/create_dummy_models.py:356–508

Create a processor for `processor_class`. If a processor is not able to be built with the original arguments, this method tries to change the arguments and call itself recursively, by inferring a new `config_class` or a new `processor_class` from another one, in order to find a checkpoi

(config_class, processor_class, allow_no_checkpoint=False)

def build_processor(config_class, processor_class, allow_no_checkpoint=False):
    """Create a processor for `processor_class`.

    If a processor is not able to be built with the original arguments, this method tries to change the arguments and
    call itself recursively, by inferring a new `config_class` or a new `processor_class` from another one, in order to
    find a checkpoint containing the necessary files to build a processor.

    The processor is not saved here. Instead, it will be saved in `convert_processors` after further changes in
    `convert_processors`. For each model architecture`, a copy will be created and saved along the built model.
    """
    # Currently, this solely uses the docstring in the source file of `config_class` to find a checkpoint.
    checkpoint = get_checkpoint_from_config_class(config_class)

    # New method that is more robust to get checkpoints!

    if checkpoint is None and not processor_class.__name__.startswith("Auto"):
        # try to get the checkpoint from the config class for `processor_class`.
        # This helps cases like `XCLIPConfig` and `VideoMAEFeatureExtractor` to find a checkpoint from `VideoMAEConfig`.
        config_class_from_processor_class = get_config_class_from_processor_class(processor_class)
        checkpoint = get_checkpoint_from_config_class(config_class_from_processor_class)

    processor = None
    try:
        revision = CHECKPOINT_REVISIONS.get(config_class.__name__)
        sub_folder = CHECKPOINT_SUBFOLDERS.get(config_class.__name__, "")
        processor = processor_class.from_pretrained(checkpoint, revision=revision, subfolder=sub_folder)
    except Exception as e:
        logger.error(f"{e.__class__.__name__}: {e}")

    # Try to get a new processor class from checkpoint. This is helpful for a checkpoint without necessary file to load
    # processor while `processor_class` is an Auto class. For example, `sew` has `Wav2Vec2Processor` in
    # `PROCESSOR_MAPPING_NAMES`, its `tokenizer_class` is `AutoTokenizer`, and the checkpoint
    # `https://huggingface.co/asapp/sew-tiny-100k` has no tokenizer file, but we can get
    # `tokenizer_class: Wav2Vec2CTCTokenizer` from the config file. (The new processor class won't be able to load from
    # `checkpoint`, but it helps this recursive method to find a way to build a processor).
    if (
        processor is None
        and checkpoint is not None
        and issubclass(processor_class, (PreTrainedTokenizerBase, AutoTokenizer))
    ):
        try:
            revision = CHECKPOINT_REVISIONS.get(config_class.__name__)
            config = AutoConfig.from_pretrained(checkpoint, revision=revision)
        except Exception as e:
            logger.error(f"{e.__class__.__name__}: {e}")
            config = None
        if config is not None:
            # TODO: sam2 (Sam2Config) from `facebook/sam2.1-hiera-tiny` will fail if we don't add `getattr(config, "tokenizer_class", None) is not None`
            # (as we get `Sam2VideoConfig` instead of `Sam2Config`)
            if getattr(config, "tokenizer_class", None) is not None and not isinstance(config, config_class):
                raise ValueError(
                    f"`config` (which is of type {config.__class__.__name__}) should be an instance of `config_class`"
                    f" ({config_class.__name__})!"
                )
            if getattr(config, "tokenizer_class", None) is not None:
                tokenizer_class = config.tokenizer_class
                new_processor_class = None
                if tokenizer_class is not None: