(
self,
num_layers: int,
num_heads: int,
d_model: int,
vocab_size: int,
dropout: float = 0.2,
dff: int = 2048, # hidden dimension of the feedforward network
maximum_position_encoding: int = 50,
)
| 146 | """Transformer Decoder""" |
| 147 | |
| 148 | def __init__( |
| 149 | self, |
| 150 | num_layers: int, |
| 151 | num_heads: int, |
| 152 | d_model: int, |
| 153 | vocab_size: int, |
| 154 | dropout: float = 0.2, |
| 155 | dff: int = 2048, # hidden dimension of the feedforward network |
| 156 | maximum_position_encoding: int = 50, |
| 157 | ) -> None: |
| 158 | super(Decoder, self).__init__() |
| 159 | self.num_layers = num_layers |
| 160 | self.d_model = d_model |
| 161 | |
| 162 | self.layer_norm_input = nn.LayerNorm(d_model, eps=1e-5) |
| 163 | self.layer_norm_masked_attention = nn.LayerNorm(d_model, eps=1e-5) |
| 164 | self.layer_norm_attention = nn.LayerNorm(d_model, eps=1e-5) |
| 165 | self.layer_norm_output = nn.LayerNorm(d_model, eps=1e-5) |
| 166 | |
| 167 | self.dropout = nn.Dropout(dropout) |
| 168 | self.embed = nn.Embedding(vocab_size, d_model) |
| 169 | self.positional_encoding = PositionalEncoding(d_model, dropout, maximum_position_encoding) |
| 170 | |
| 171 | self.attention = nn.ModuleList([ |
| 172 | MultiHeadAttention(num_heads, d_model, dropout) for _ in range(self.num_layers) |
| 173 | ]) |
| 174 | self.source_attention = nn.ModuleList([ |
| 175 | MultiHeadAttention(num_heads, d_model, dropout) for _ in range(self.num_layers) |
| 176 | ]) |
| 177 | self.position_feed_forward = nn.ModuleList([ |
| 178 | PositionwiseFeedForward(d_model, dff, dropout) for _ in range(self.num_layers) |
| 179 | ]) |
| 180 | |
| 181 | def forward( |
| 182 | self, |
no test coverage detected