Clear allocated cacheKV
(self, profile=False)
| 407 | self.attn_backends.append(attn_backend) |
| 408 | |
| 409 | def clear_mtp_cache(self, profile=False): |
| 410 | """ |
| 411 | Clear allocated cacheKV |
| 412 | """ |
| 413 | create_cache_tensor = profile or not ( |
| 414 | self.fd_config.cache_config.num_cpu_blocks > 0 |
| 415 | or self.fd_config.cache_config.kvcache_storage_backend |
| 416 | or self.fd_config.scheduler_config.splitwise_role != "mixed" |
| 417 | ) |
| 418 | if not create_cache_tensor: |
| 419 | for name, tensor in self.cache_kvs_map.items(): |
| 420 | unset_data_ipc(tensor, name, True, False) |
| 421 | self.cache_kvs_map.clear() |
| 422 | del self.model_inputs["caches"] |
| 423 | if self.forward_meta is not None: |
| 424 | del self.forward_meta.caches |
| 425 | |
| 426 | def update_mtp_block_num(self, num_gpu_blocks) -> None: |
| 427 | """ |