The Entrance of model execute. Args: model_forward_batch: 'Request' contains information related to prompt and is an abstract class at the server level, which is too granular for ModelRunner. We plan to replace it with 'ModelForwardBatch'.
(
self,
model_forward_batch: Optional[List[Request]] = None,
num_running_requests: int = None,
is_dummy_run: bool = False,
in_capturing: bool = False,
)
| 1523 | logger.info(f"SOT warmup took {time.perf_counter() - start_time} seconds") |
| 1524 | |
| 1525 | def execute_model( |
| 1526 | self, |
| 1527 | model_forward_batch: Optional[List[Request]] = None, |
| 1528 | num_running_requests: int = None, |
| 1529 | is_dummy_run: bool = False, |
| 1530 | in_capturing: bool = False, |
| 1531 | ) -> Optional[ModelRunnerOutput]: |
| 1532 | """ |
| 1533 | The Entrance of model execute. |
| 1534 | Args: |
| 1535 | model_forward_batch: 'Request' contains information related to prompt and is an abstract |
| 1536 | class at the server level, which is too granular for ModelRunner. |
| 1537 | We plan to replace it with 'ModelForwardBatch'. |
| 1538 | num_running_requests: batch_size |
| 1539 | intermediate_tensors: |
| 1540 | """ |
| 1541 | # 0. set debug level |
| 1542 | # self._set_debug_level(0x1, model_forward_batch, is_dummy_run) |
| 1543 | with kv_signal_sender_context_manager(self.pd_disaggregation_mode) as sender: |
| 1544 | |
| 1545 | self.share_inputs["kv_signal_sender"] = sender |
| 1546 | # 1. Prepare inputs of model and decoder. |
| 1547 | self._prepare_inputs(is_dummy_run=is_dummy_run) |
| 1548 | |
| 1549 | if is_dummy_run: |
| 1550 | self.forward_meta.step_use_cudagraph = in_capturing and self.forward_meta.step_use_cudagraph |
| 1551 | # 2. Padding inputs for cuda grph |
| 1552 | self.padding_cudagraph_inputs() |
| 1553 | |
| 1554 | # NOTE(wufeisheng): If `not_need_stop`` is False, it means the current worker is in an idle state. |
| 1555 | # This logic is not used in TP (Tensor Parallelism) mode. However, in EP (Expert Parallelism) mode, |
| 1556 | # when there is data on other runner, the current runner is required to execute part of the model. |
| 1557 | if not self.not_need_stop() and not is_dummy_run: |
| 1558 | self._execute_empty_input(self.forward_meta) |
| 1559 | return None |
| 1560 | |
| 1561 | # 2. Padding inputs for cuda grph |
| 1562 | |
| 1563 | # 3. Execute model |
| 1564 | if self.enable_mm: |
| 1565 | model_output = self.model( |
| 1566 | self.share_inputs["ids_remove_padding"], self.share_inputs["image_features"], self.forward_meta |
| 1567 | ) |
| 1568 | else: |
| 1569 | model_output = self.model( |
| 1570 | ids_remove_padding=self.share_inputs["ids_remove_padding"], |
| 1571 | forward_meta=self.forward_meta, |
| 1572 | ) |
| 1573 | if self.use_cudagraph: |
| 1574 | model_output = model_output[: self.real_token_num] |
| 1575 | hidden_states = xpu_process_output( |
| 1576 | model_output, self.share_inputs["cum_offsets"], self.forward_meta, self.share_inputs |
| 1577 | ) |
| 1578 | # 4. Compute logits, Sample |
| 1579 | logits = self.model.compute_logits(hidden_states) |
| 1580 | sampler_output = None |
| 1581 | if not self.speculative_decoding: |
| 1582 | sampler_output = self.sampler(logits, self.sampling_metadata) |
no test coverage detected