| 86 | |
| 87 | |
| 88 | class XPUModelRunner(ModelRunnerBase): |
| 89 | """ """ |
| 90 | |
| 91 | def __init__( |
| 92 | self, |
| 93 | fd_config: FDConfig, |
| 94 | device: str, # logic device |
| 95 | device_id: int, # physical device id |
| 96 | rank: int, |
| 97 | local_rank: int, |
| 98 | ): |
| 99 | super().__init__(fd_config=fd_config, device=device) |
| 100 | self.enable_mm = self.model_config.enable_mm |
| 101 | self.rank = rank |
| 102 | self.local_rank = local_rank |
| 103 | self.device_id = device_id |
| 104 | self.enable_early_stop = self.fd_config.early_stop_config.enable_early_stop |
| 105 | self.enable_logprob = fd_config.model_config.enable_logprob |
| 106 | self.ori_vocab_size = self.fd_config.model_config.ori_vocab_size |
| 107 | self.max_logprobs = ( |
| 108 | self.ori_vocab_size if fd_config.model_config.max_logprobs == -1 else fd_config.model_config.max_logprobs |
| 109 | ) |
| 110 | |
| 111 | # VL model config: |
| 112 | if self.enable_mm: |
| 113 | self._init_image_preprocess() |
| 114 | |
| 115 | self.amp_black = [ |
| 116 | "reduce_sum", |
| 117 | "c_softmax_with_cross_entropy", |
| 118 | "elementwise_div", |
| 119 | "sin", |
| 120 | "cos", |
| 121 | "sort", |
| 122 | "multinomial", |
| 123 | ] |
| 124 | self.amp_white = [ |
| 125 | "lookup_table", |
| 126 | "lookup_table_v2", |
| 127 | "flash_attn", |
| 128 | "matmul", |
| 129 | "matmul_v2", |
| 130 | "fused_gemm_epilogue", |
| 131 | ] |
| 132 | if self.cache_config.max_encoder_cache > 0: |
| 133 | self.encoder_cache: dict[str, paddle.Tensor] = {} |
| 134 | else: |
| 135 | self.encoder_cache = None |
| 136 | |
| 137 | self.device_id = device_id |
| 138 | self.speculative_method = self.fd_config.speculative_config.method |
| 139 | self.speculative_decoding = self.speculative_method is not None |
| 140 | |
| 141 | # used by SamplingMetadata |
| 142 | self.enable_logprob = fd_config.model_config.enable_logprob # fd_config.model_config.enable_logprob |
| 143 | self.enable_early_stop = self.fd_config.early_stop_config.enable_early_stop |
| 144 | |
| 145 | # Sampler |