| 135 | |
| 136 | |
| 137 | class MLP(nn.Module): |
| 138 | def __init__(self, config: CLIPTextConfig): |
| 139 | super().__init__() |
| 140 | self.config = config |
| 141 | self.activation_fn = quick_gelu |
| 142 | self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) |
| 143 | self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) |
| 144 | |
| 145 | def __call__(self, x: mx.array) -> mx.array: |
| 146 | x = self.activation_fn(self.fc1(x)) |
| 147 | x = self.fc2(x) |
| 148 | return x |
| 149 | |
| 150 | |
| 151 | class EncoderLayer(nn.Module): |