| 22 | |
| 23 | |
| 24 | class DCUModelRunner(GPUModelRunner): |
| 25 | def __init__( |
| 26 | self, |
| 27 | fd_config: FDConfig, |
| 28 | device: str, # logic device |
| 29 | device_id: int, # physical device id |
| 30 | rank: int, |
| 31 | local_rank: int, |
| 32 | ): |
| 33 | super(DCUModelRunner, self).__init__( |
| 34 | fd_config=fd_config, device=device, device_id=device_id, rank=rank, local_rank=local_rank |
| 35 | ) |
| 36 | |
| 37 | def initialize_forward_meta(self): |
| 38 | """ |
| 39 | Initialize forward meta and attention meta data |
| 40 | """ |
| 41 | # Initialize forward meta |
| 42 | self.forward_meta = DCUForwardMeta( |
| 43 | input_ids=self.share_inputs["input_ids"], |
| 44 | ids_remove_padding=self.share_inputs["ids_remove_padding"], |
| 45 | rotary_embs=self.share_inputs["rope_emb"], |
| 46 | attn_backend=self.attn_backends[0], |
| 47 | decoder_batch_ids=self.share_inputs["decoder_batch_ids"], |
| 48 | decoder_tile_ids_per_batch=self.share_inputs["decoder_tile_ids_per_batch"], |
| 49 | decoder_num_blocks_cpu=self.share_inputs["decoder_num_blocks_cpu"], |
| 50 | max_len_tensor_cpu=self.share_inputs["max_len_tensor_cpu"], |
| 51 | seq_lens_encoder=self.share_inputs["seq_lens_encoder"], |
| 52 | seq_lens_decoder=self.share_inputs["seq_lens_decoder"], |
| 53 | seq_lens_this_time=self.share_inputs["seq_lens_this_time"], |
| 54 | batch_id_per_token=self.share_inputs["batch_id_per_token"], |
| 55 | cum_offsets=self.share_inputs["cum_offsets"], |
| 56 | cu_seqlens_q=self.share_inputs["cu_seqlens_q"], |
| 57 | cu_seqlens_k=self.share_inputs["cu_seqlens_k"], |
| 58 | block_tables=self.share_inputs["block_tables"], |
| 59 | caches=self.share_inputs["caches"], |
| 60 | ) |
| 61 | |
| 62 | # Update Batch type for cuda graph |
| 63 | only_decode_batch = True |
| 64 | prefill_exists = None |
| 65 | # mix ep in single node |
| 66 | if self.fd_config.parallel_config.use_ep and self.fd_config.scheduler_config.splitwise_role == "mixed": |
| 67 | only_decode_batch_list = [] |
| 68 | prefill_exists = self.exist_prefill() |
| 69 | paddle.distributed.all_gather_object(only_decode_batch_list, not prefill_exists) |
| 70 | only_decode_batch = all(only_decode_batch_list) |
| 71 | self.fd_config.model_config.moe_phase.phase = "decode" if only_decode_batch else "prefill" |
| 72 | |
| 73 | self.forward_meta.step_use_cudagraph = ( |
| 74 | self.use_cudagraph |
| 75 | and only_decode_batch |
| 76 | and not (prefill_exists if prefill_exists is not None else self.exist_prefill()) |
| 77 | ) |
| 78 | |
| 79 | # Initialzie attention meta data |
| 80 | for attn_backend in self.attn_backends: |
| 81 | attn_backend.init_attention_metadata(self.forward_meta) |