| 14 | |
| 15 | |
| 16 | class TransformerEncoderLayer(nn.Module): |
| 17 | def __init__(self, hidden_size, dropout, kernel_size=None, num_heads=2, norm='ln'): |
| 18 | super().__init__() |
| 19 | self.hidden_size = hidden_size |
| 20 | self.dropout = dropout |
| 21 | self.num_heads = num_heads |
| 22 | self.op = EncSALayer( |
| 23 | hidden_size, num_heads, dropout=dropout, |
| 24 | attention_dropout=0.0, relu_dropout=dropout, |
| 25 | kernel_size=kernel_size |
| 26 | if kernel_size is not None else hparams['enc_ffn_kernel_size'], |
| 27 | padding=hparams['ffn_padding'], |
| 28 | norm=norm, act=hparams['ffn_act']) |
| 29 | |
| 30 | def forward(self, x, **kwargs): |
| 31 | return self.op(x, **kwargs) |
| 32 | |
| 33 | |
| 34 | ###################### |