The Entrance of model execute. Args: model_forward_batch: 'Request' contains information related to prompt and is an abstract class at the server level, which is too granular for ModelRunner. We plan to replace it with 'ModelForwardBatch'.
(
self,
model_forward_batch: Optional[List[Request]] = None,
)
| 1552 | logger.info(f"Cuda Graph capturing took {time_after_capture - time_before_capture} seconds") |
| 1553 | |
| 1554 | def execute_model( |
| 1555 | self, |
| 1556 | model_forward_batch: Optional[List[Request]] = None, |
| 1557 | ) -> Optional[ModelRunnerOutput]: |
| 1558 | """ |
| 1559 | The Entrance of model execute. |
| 1560 | Args: |
| 1561 | model_forward_batch: 'Request' contains information related to prompt and is an abstract |
| 1562 | class at the server level, which is too granular for ModelRunner. |
| 1563 | We plan to replace it with 'ModelForwardBatch'. |
| 1564 | intermediate_tensors: |
| 1565 | """ |
| 1566 | if (self.parallel_config.use_ep) and (not self.not_need_stop()): |
| 1567 | time.sleep(0.001) |
| 1568 | return None |
| 1569 | # # 1. Prepare inputs of model and decoder. |
| 1570 | start_time = time.time() |
| 1571 | self._prepare_inputs() |
| 1572 | # self.share_inputs["ids_remove_padding"].cpu() |
| 1573 | # # 2. Padding inputs for cuda grph |
| 1574 | end_time = time.time() |
| 1575 | execution_time = (end_time - start_time) * 1000 |
| 1576 | real_bs = self.share_inputs["ids_remove_padding"].shape[0] |
| 1577 | hpu_model_runner_profile_logger.info(f"_prepare_inputs time(ms): {execution_time}, BT={real_bs}") |
| 1578 | start_time = time.time() |
| 1579 | # # 3. Execute model |
| 1580 | model_output = self.model(self.share_inputs["ids_remove_padding"], self.forward_meta) |
| 1581 | if self.is_hpu_perf_breakdown_sync_mode: |
| 1582 | model_output.cpu() |
| 1583 | end_time = time.time() |
| 1584 | execution_time = (end_time - start_time) * 1000 |
| 1585 | hpu_model_runner_profile_logger.info( |
| 1586 | f"Model execution time(ms): {execution_time}, BT={real_bs}, block_list_encoder_shape={self.share_inputs['block_list_encoder'].shape}, block_indices_encoder_shape={self.share_inputs['block_indices_encoder'].shape}" |
| 1587 | ) |
| 1588 | hpu_model_runner_profile_logger.info( |
| 1589 | f"Model execution time(ms): {execution_time}, BT={real_bs}, block_list_decoder_shape={self.share_inputs['block_list_decoder'].shape}, block_indices_decoder_shape={self.share_inputs['block_indices_decoder'].shape}" |
| 1590 | ) |
| 1591 | |
| 1592 | start_time = time.time() |
| 1593 | start_time0 = time.time() |
| 1594 | if self.forward_meta.total_batch_encoder > 0 and self.forward_meta.total_batch_decoder > 0: |
| 1595 | hiddden_states = rebuild_padding_mixed_v3_1( |
| 1596 | model_output, |
| 1597 | self.forward_meta.batch_ids_encoder, |
| 1598 | self.forward_meta.total_batch_encoder, |
| 1599 | self.forward_meta.batch_ids_decoder, |
| 1600 | self.forward_meta.total_batch_decoder, |
| 1601 | self.forward_meta.seq_lens_encoder, |
| 1602 | ) |
| 1603 | elif self.forward_meta.total_batch_encoder > 0: |
| 1604 | hiddden_states = rebuild_padding_v3_1( |
| 1605 | model_output, |
| 1606 | self.forward_meta.batch_ids_encoder, |
| 1607 | self.forward_meta.total_batch_encoder, |
| 1608 | self.forward_meta.seq_lens_encoder, |
| 1609 | True, |
| 1610 | ) |
| 1611 | elif self.forward_meta.total_batch_decoder > 0: |
no test coverage detected