| 135 | |
| 136 | |
| 137 | class CLIPEncoder(Module): |
| 138 | |
| 139 | def __init__(self, hidden_size, num_attention_heads, |
| 140 | max_position_embeddings, norm_epsilon, intermediate_size, |
| 141 | hidden_act, num_hidden_layers, mapping: Mapping, dtype): |
| 142 | super().__init__() |
| 143 | self.hidden_size = hidden_size |
| 144 | self.dtype = dtype |
| 145 | self.mapping = mapping |
| 146 | |
| 147 | self.layers = ModuleList([ |
| 148 | CLIPEncoderLayer(hidden_size=self.hidden_size, |
| 149 | num_attention_heads=num_attention_heads, |
| 150 | max_position_embeddings=max_position_embeddings, |
| 151 | norm_epsilon=norm_epsilon, |
| 152 | intermediate_size=intermediate_size, |
| 153 | hidden_act=hidden_act, |
| 154 | mapping=self.mapping, |
| 155 | dtype=self.dtype) for _ in range(num_hidden_layers) |
| 156 | ]) |
| 157 | |
| 158 | def forward(self, inputs_embeds): |
| 159 | |
| 160 | hidden_states = inputs_embeds |
| 161 | for layer in self.layers: |
| 162 | hidden_states = layer(hidden_states) |
| 163 | |
| 164 | return hidden_states |
| 165 | |
| 166 | |
| 167 | class CLIPVisionTransformer(Module): |