| 23 | |
| 24 | |
| 25 | class Encoder(nn.Module): |
| 26 | def __init__( |
| 27 | self, |
| 28 | d_model: int, |
| 29 | mha: AttentionLayer, |
| 30 | d_hidden: int, |
| 31 | dropout: float = 0, |
| 32 | channel_wise=False, |
| 33 | ): |
| 34 | super(Encoder, self).__init__() |
| 35 | |
| 36 | self.channel_wise = channel_wise |
| 37 | if self.channel_wise: |
| 38 | self.conv = torch.nn.Conv1d( |
| 39 | in_channels=d_model, |
| 40 | out_channels=d_model, |
| 41 | kernel_size=1, |
| 42 | stride=1, |
| 43 | padding=0, |
| 44 | padding_mode="reflect", |
| 45 | ) |
| 46 | self.MHA = mha |
| 47 | self.feedforward = FeedForward(d_model=d_model, d_hidden=d_hidden) |
| 48 | self.dropout = torch.nn.Dropout(p=dropout) |
| 49 | self.layerNormal_1 = torch.nn.LayerNorm(d_model) |
| 50 | self.layerNormal_2 = torch.nn.LayerNorm(d_model) |
| 51 | |
| 52 | def forward(self, x): |
| 53 | residual = x |
| 54 | q = residual |
| 55 | if self.channel_wise: |
| 56 | x_r = self.conv(x.permute(0, 2, 1)).transpose(1, 2) |
| 57 | k = x_r |
| 58 | v = x_r |
| 59 | else: |
| 60 | k = residual |
| 61 | v = residual |
| 62 | x, score = self.MHA(q, k, v, attn_mask=None) |
| 63 | x = self.dropout(x) |
| 64 | x = self.layerNormal_1(x + residual) |
| 65 | |
| 66 | residual = x |
| 67 | x = self.feedforward(residual) |
| 68 | x = self.dropout(x) |
| 69 | x = self.layerNormal_2(x + residual) |
| 70 | |
| 71 | return x, score |
| 72 | |
| 73 | |
| 74 | class Model(nn.Module): |