(args)
| 1026 | |
| 1027 | |
| 1028 | def build_phi4mm_engine(args): |
| 1029 | logger.warning( |
| 1030 | "Skipping TRT engine build for Phi-4-multimodal encoder. MultimodalModelRunner will use PyTorch vision & audio encoder. Flash/SDPA attention in CLIP encoder is not compatible with torch.onnx.export and eager attention is unstable in PyTorch." |
| 1031 | ) |
| 1032 | |
| 1033 | # Dump config.json needed by model runner |
| 1034 | config_args = { |
| 1035 | "builder_config": { |
| 1036 | "precision": torch_dtype_to_str(torch.float16), |
| 1037 | "model_type": "phi-4-multimodal", |
| 1038 | } |
| 1039 | } |
| 1040 | os.makedirs(os.path.join(args.output_dir, "vision"), exist_ok=True) |
| 1041 | os.makedirs(os.path.join(args.output_dir, "audio"), exist_ok=True) |
| 1042 | to_json_file(config_args, |
| 1043 | os.path.join(args.output_dir, "vision", "config.json")) |
| 1044 | to_json_file(config_args, |
| 1045 | os.path.join(args.output_dir, "audio", "config.json")) |
| 1046 | return |
| 1047 | |
| 1048 | # Following code works ok with eager mode attention. Leaving it here so that it could |
| 1049 | # be used once issues in torch / onnx mentioned above resolved. |
| 1050 | processor = AutoProcessor.from_pretrained(args.model_path, |
| 1051 | trust_remote_code=True) |
| 1052 | raw_image = Image.new('RGB', [10, 10]) # dummy image |
| 1053 | |
| 1054 | import numpy as np |
| 1055 | audio_feature_size = 500 |
| 1056 | audio_compression_rate = 8 |
| 1057 | audio_sampling_rate = 16000 |
| 1058 | audio_len = int((audio_feature_size * audio_compression_rate + 2) * |
| 1059 | audio_sampling_rate / 100) |
| 1060 | raw_audio = (np.zeros(audio_len), audio_sampling_rate) # dummy audio |
| 1061 | |
| 1062 | inputs = processor(text="<|image_1|><|audio_1|>\ndummy", |
| 1063 | images=[raw_image], |
| 1064 | audios=[raw_audio], |
| 1065 | return_tensors="pt") |
| 1066 | |
| 1067 | img_embeds = inputs['input_image_embeds'].to(args.device, torch.float16) |
| 1068 | img_attention_mask = inputs['image_attention_mask'].to( |
| 1069 | args.device, torch.bool) |
| 1070 | img_embeds = img_embeds.flatten(0, 1) # (2, 3, 448, 448) |
| 1071 | img_attention_mask = img_attention_mask.flatten(0, 1) # (2, 32, 32) |
| 1072 | |
| 1073 | aud_embeds = inputs['input_audio_embeds'].to(args.device, |
| 1074 | torch.float16) # (1, 4000, 80) |
| 1075 | aud_len, aud_dim = aud_embeds.shape[1:] |
| 1076 | aud_embeds = torch.cat( |
| 1077 | [aud_embeds, |
| 1078 | aud_embeds.new_zeros(1, 4000 - aud_len, aud_dim)], dim=1) |
| 1079 | aud_attention_mask = torch.ones(1, aud_embeds.shape[1]).to( |
| 1080 | args.device, torch.bool) |
| 1081 | aud_attention_mask[0, aud_len:] = 0 |
| 1082 | |
| 1083 | class Phi4VisionWrapper(torch.nn.Module): |
| 1084 | |
| 1085 | def __init__(self, vision_model): |
no test coverage detected