Function convert_to_conversation_message

tensorrt_llm/inputs/utils.py:670–765

(
        prompt: str,
        media: Union[Any, List[Any]],
        modality: str,
        is_embedding: bool = False,
    )

        device: str = "cpu") -> List[dict[str, Union[str, torch.Tensor]]]:

    def convert_to_conversation_message(
        prompt: str,
        media: Union[Any, List[Any]],
        modality: str,
        is_embedding: bool = False,
    ) -> ConversationMessage:
        if isinstance(media, str):
            media = [media]
        if modality in ["image", "multiple_image"]:
            if is_embedding:
                _load = lambda mm: mm

                # each mm_embedding corresponds to each image placeholder
                if not isinstance(media, list):
                    media = [media]
            else:
                _load = lambda mm: load_image(
                    mm, format=image_data_format, device=device)

            mm_data = [
                MultimodalData(modality=modality,
                               data=_load(mm),
                               is_embedding=is_embedding) for mm in media
            ]
        elif modality == "video":
            if is_embedding:
                raise ValueError(
                    "External embedding is not supported for video modality yet."
                )
            mm_data = [
                MultimodalData(
                    modality=modality,
                    data=load_video(i,
                                    num_frames,
                                    format=image_data_format,
                                    device=device),
                    is_embedding=False,
                ) for i in media
            ]
        elif modality == "audio":
            if is_embedding:
                raise ValueError(
                    "External embedding is not supported for audio modality yet."
                )
            mm_data = [
                MultimodalData(
                    modality=modality,
                    data=load_audio(i, device=device),
                    is_embedding=False,
                ) for i in media
            ]
        elif modality == "image_audio":
            if is_embedding:
                raise ValueError(
                    "External embedding is not supported for image_audio modality yet."
                )
            # Use different load_xxx functions to match the modality.
            mm_data = []