Execute a forward pass with dummy inputs to profile the memory usage of the model
(self)
| 2705 | |
| 2706 | @profile_run_guard(True) |
| 2707 | def profile_run(self) -> None: |
| 2708 | """Execute a forward pass with dummy inputs to profile the memory usage of the model""" |
| 2709 | # Initialize kv cache for profile run. After profile run kv cache will be reset. |
| 2710 | # TODO(gongshaotian): Optimize the management logic of kvcache |
| 2711 | self.num_gpu_blocks = self.cache_config.total_block_num |
| 2712 | self.initialize_kv_cache(profile=True) |
| 2713 | if self.speculative_method in ["mtp"]: |
| 2714 | self.proposer.initialize_kv_cache(main_model_num_blocks=self.num_gpu_blocks, profile=True) |
| 2715 | |
| 2716 | # 1. Profile with multimodal encoder & encoder cache |
| 2717 | |
| 2718 | # 2. Dummy run |
| 2719 | num_tokens = self.fd_config.get_max_chunk_tokens() |
| 2720 | logger.info( |
| 2721 | f"Dummy run with {num_tokens} tokens, mm_max_tokens_per_item: {self.model_config.mm_max_tokens_per_item}" |
| 2722 | ) |
| 2723 | self._dummy_run( |
| 2724 | num_tokens=num_tokens, |
| 2725 | batch_size=self.scheduler_config.max_num_seqs, |
| 2726 | ) |
| 2727 | |
| 2728 | # 3. gc |
| 2729 | if self.speculative_method in ["mtp"]: |
| 2730 | self.proposer.clear_mtp_cache(profile=True) |
| 2731 | self.clear_cache(profile=True) |
| 2732 | |
| 2733 | def update_share_input_block_num(self, num_gpu_blocks: int) -> None: |
| 2734 | """ |
nothing calls this directly
no test coverage detected