(self, c, num_heads, dropout, attention_dropout=0.1, relu_dropout=0.1, kernel_size=9, act='gelu')
| 590 | |
| 591 | class DecSALayer(nn.Module): |
| 592 | def __init__(self, c, num_heads, dropout, attention_dropout=0.1, relu_dropout=0.1, kernel_size=9, act='gelu'): |
| 593 | super().__init__() |
| 594 | self.c = c |
| 595 | self.dropout = dropout |
| 596 | self.layer_norm1 = LayerNorm(c) |
| 597 | self.self_attn = MultiheadAttention( |
| 598 | c, num_heads, self_attention=True, dropout=attention_dropout, bias=False |
| 599 | ) |
| 600 | self.layer_norm2 = LayerNorm(c) |
| 601 | self.encoder_attn = MultiheadAttention( |
| 602 | c, num_heads, encoder_decoder_attention=True, dropout=attention_dropout, bias=False, |
| 603 | ) |
| 604 | self.layer_norm3 = LayerNorm(c) |
| 605 | self.ffn = TransformerFFNLayer( |
| 606 | c, 4 * c, padding='LEFT', kernel_size=kernel_size, dropout=relu_dropout, act=act) |
| 607 | |
| 608 | def forward( |
| 609 | self, |
nothing calls this directly
no test coverage detected