A transformer encoder layer with (the original BERT) post-normalization.
| 9 | |
| 10 | |
| 11 | class TransformerEncoderLayer(nn.Module): |
| 12 | """ |
| 13 | A transformer encoder layer with (the original BERT) post-normalization. |
| 14 | """ |
| 15 | |
| 16 | def __init__( |
| 17 | self, |
| 18 | dims: int, |
| 19 | num_heads: int, |
| 20 | mlp_dims: Optional[int] = None, |
| 21 | layer_norm_eps: float = 1e-12, |
| 22 | ): |
| 23 | super().__init__() |
| 24 | mlp_dims = mlp_dims or dims * 4 |
| 25 | self.attention = nn.MultiHeadAttention(dims, num_heads, bias=True) |
| 26 | self.ln1 = nn.LayerNorm(dims, eps=layer_norm_eps) |
| 27 | self.ln2 = nn.LayerNorm(dims, eps=layer_norm_eps) |
| 28 | self.linear1 = nn.Linear(dims, mlp_dims) |
| 29 | self.linear2 = nn.Linear(mlp_dims, dims) |
| 30 | self.gelu = nn.GELU() |
| 31 | |
| 32 | def __call__(self, x, mask): |
| 33 | attention_out = self.attention(x, x, x, mask) |
| 34 | add_and_norm = self.ln1(x + attention_out) |
| 35 | |
| 36 | ff = self.linear1(add_and_norm) |
| 37 | ff_gelu = self.gelu(ff) |
| 38 | ff_out = self.linear2(ff_gelu) |
| 39 | x = self.ln2(ff_out + add_and_norm) |
| 40 | |
| 41 | return x |
| 42 | |
| 43 | |
| 44 | class TransformerEncoder(nn.Module): |