| 14 | """ |
| 15 | |
| 16 | def __init__( |
| 17 | self, |
| 18 | dims: int, |
| 19 | num_heads: int, |
| 20 | mlp_dims: Optional[int] = None, |
| 21 | layer_norm_eps: float = 1e-12, |
| 22 | ): |
| 23 | super().__init__() |
| 24 | mlp_dims = mlp_dims or dims * 4 |
| 25 | self.attention = nn.MultiHeadAttention(dims, num_heads, bias=True) |
| 26 | self.ln1 = nn.LayerNorm(dims, eps=layer_norm_eps) |
| 27 | self.ln2 = nn.LayerNorm(dims, eps=layer_norm_eps) |
| 28 | self.linear1 = nn.Linear(dims, mlp_dims) |
| 29 | self.linear2 = nn.Linear(mlp_dims, dims) |
| 30 | self.gelu = nn.GELU() |
| 31 | |
| 32 | def __call__(self, x, mask): |
| 33 | attention_out = self.attention(x, x, x, mask) |