(model_type: str,
conversation: list[ConversationMessage],
mm_placeholder_counts: list[dict[str, int]])
| 562 | |
| 563 | |
| 564 | def handle_placeholder_exceptions(model_type: str, |
| 565 | conversation: list[ConversationMessage], |
| 566 | mm_placeholder_counts: list[dict[str, int]]): |
| 567 | if model_type == "llava_next": |
| 568 | # we need to convert the flattened content back to conversation format |
| 569 | for conv, mm_placeholder_count in zip(conversation, |
| 570 | mm_placeholder_counts): |
| 571 | conv["content"] = [{"type": "text", "text": conv["content"]}, \ |
| 572 | *[{"type": "image"} for _ in range(mm_placeholder_count['<image>'])]] |
| 573 | elif model_type == "NemotronH_Nano_VL_V2": |
| 574 | # There are divergences between trtllm and vllm on how to handle the placeholders. |
| 575 | # For now, we will use this exception to handle with the divergences in TRTLLM. |
| 576 | # In the near future, we will remove this placeholder exception and use dict format as vllm does. |
| 577 | for conv, mm_placeholder_count in zip(conversation, |
| 578 | mm_placeholder_counts): |
| 579 | if '<image>' not in mm_placeholder_count and '<video>' not in mm_placeholder_count: |
| 580 | # Skip if no image or video placeholders. |
| 581 | continue |
| 582 | |
| 583 | # Contents from all kinds of roles will be handled. |
| 584 | content = [] |
| 585 | content.append({"type": "text", "text": conv["content"]}) |
| 586 | # Extend image/video placeholders so that the chat_template can be applied correctly. |
| 587 | if '<image>' in mm_placeholder_count: |
| 588 | content.extend([{ |
| 589 | "type": "image" |
| 590 | } for _ in range(mm_placeholder_count['<image>'])]) |
| 591 | if '<video>' in mm_placeholder_count: |
| 592 | content.extend([{ |
| 593 | "type": "video" |
| 594 | } for _ in range(mm_placeholder_count['<video>'])]) |
| 595 | conv["content"] = content |
| 596 | else: |
| 597 | raise ValueError(f"This path should not be reached for: {model_type}") |
| 598 | return conversation |
| 599 | |
| 600 | |
| 601 | def apply_chat_template( |
no test coverage detected