| 178 | class Phi3Model(Module): |
| 179 | |
| 180 | def __init__(self, config: PretrainedConfig): |
| 181 | super().__init__() |
| 182 | self.vocab_embedding = Embedding(num_embeddings=config.vocab_size, |
| 183 | embedding_dim=config.hidden_size, |
| 184 | dtype=config.dtype) |
| 185 | |
| 186 | self.layers = DecoderLayerList(Phi3DecoderLayer, config) |
| 187 | self.small_variant = config.architecture == "Phi3SmallForCausalLM" |
| 188 | self.moe_variant = config.architecture == "PhiMoEForCausalLM" |
| 189 | if self.small_variant or self.moe_variant: |
| 190 | self.ln_f = LayerNorm(normalized_shape=config.hidden_size, |
| 191 | eps=config.norm_epsilon, |
| 192 | dtype=config.dtype) |
| 193 | if self.small_variant: |
| 194 | self.mup_embedding_multiplier = config.mup_embedding_multiplier |
| 195 | else: |
| 196 | self.ln_f = RmsNorm(normalized_shape=config.hidden_size, |
| 197 | eps=config.norm_epsilon, |
| 198 | dtype=config.dtype) |
| 199 | |
| 200 | def forward( |
| 201 | self, |