Execute a forward pass with dummy inputs to profile the memory usage of the model.
(self)
| 1739 | raise ValueError(f"{type(self.model)} has no attribute 'empty_input_forward") |
| 1740 | |
| 1741 | def profile_run(self) -> None: |
| 1742 | """Execute a forward pass with dummy inputs to profile the memory usage of the model.""" |
| 1743 | |
| 1744 | # Initialize kv cache for profile run. After profile run kv cache will be reset. |
| 1745 | # TODO(gongshaotian): Optimize the management logic of kvcache |
| 1746 | self.num_gpu_blocks = self.cache_config.total_block_num |
| 1747 | self.initialize_kv_cache() |
| 1748 | |
| 1749 | # 1. Profile with multimodal encoder & encoder cache |
| 1750 | |
| 1751 | # 2. Dummy run |
| 1752 | self._dummy_run( |
| 1753 | num_tokens=self.scheduler_config.max_num_batched_tokens, |
| 1754 | batch_size=min(self.scheduler_config.max_num_seqs, 3), |
| 1755 | ) |
| 1756 | |
| 1757 | # 3. gc |
| 1758 | self.clear_cache() |
| 1759 | |
| 1760 | if self.speculative_method in ["mtp"]: |
| 1761 | self.proposer.clear_dummy_input() |
| 1762 | |
| 1763 | def update_share_input_block_num(self, num_gpu_blocks: int) -> None: |
| 1764 | """ |
nothing calls this directly
no test coverage detected