Function find_code_and_splits

utils/check_copies.py:498–538

Find the code of an object (specified by `object_name`) and split it into blocks. Args: object_name (`str`): The name of the object, e.g. `transformers.models.bert.modeling_bert.BertAttention` or `tests.models.llama.test_modeling_llama.LlamaModelTest.test_config`

(object_name: str, base_path: str, buffer: dict | None = None)

def find_code_and_splits(object_name: str, base_path: str, buffer: dict | None = None):
    """Find the code of an object (specified by `object_name`) and split it into blocks.

    Args:
        object_name (`str`):
            The name of the object, e.g. `transformers.models.bert.modeling_bert.BertAttention` or
            `tests.models.llama.test_modeling_llama.LlamaModelTest.test_config`.
        base_path (`str`):
            The path to the base directory within which the search will be performed. It could be either
            `TRANSFORMERS_PATH` or `MODEL_TEST_PATH`.
        buffer (`dict`, *optional*):
            The buffer used to store the previous results in order to speed up the process.

    Returns:
        lines (`List[str]`):
            The lines of the whole file where the object is defined.
        code (`str`):
            The object's code.
        code_splits (`List[Tuple[str, int, int]]`):
            `code` split into blocks. See `split_code_into_blocks`.
    """
    if buffer is None:
        buffer = {}

    if (object_name, base_path) in buffer:
        lines, code, code_splits = buffer[(object_name, base_path)]
    else:
        code, (lines, target_start_index, target_end_index) = find_code_in_transformers(
            object_name, base_path=base_path, return_indices=True
        )
        indent = get_indent(code)

        # Split the code into blocks
        # `indent` is the indent of the class/func definition header, but `code_splits` expects the indent level of the
        # block body.
        code_splits = split_code_into_blocks(
            lines, target_start_index, target_end_index, len(indent) + 4, backtrace=True
        )
        buffer[(object_name, base_path)] = lines, code, code_splits

    return lines, code, code_splits


_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+transformers\.(\S+\.\S+)\s*($|\S.*$)")