Next Transformer Block
| 274 | |
| 275 | |
| 276 | class NTB(nn.Module): |
| 277 | """ |
| 278 | Next Transformer Block |
| 279 | """ |
| 280 | def __init__( |
| 281 | self, in_channels, out_channels, path_dropout, stride=1, sr_ratio=1, |
| 282 | mlp_ratio=2, head_dim=32, mix_block_ratio=0.75, attn_drop=0, drop=0, |
| 283 | ): |
| 284 | super(NTB, self).__init__() |
| 285 | self.in_channels = in_channels |
| 286 | self.out_channels = out_channels |
| 287 | self.mix_block_ratio = mix_block_ratio |
| 288 | norm_func = partial(nn.BatchNorm2d, eps=NORM_EPS) |
| 289 | |
| 290 | self.mhsa_out_channels = _make_divisible(int(out_channels * mix_block_ratio), 32) |
| 291 | self.mhca_out_channels = out_channels - self.mhsa_out_channels |
| 292 | |
| 293 | self.patch_embed = PatchEmbed(in_channels, self.mhsa_out_channels, stride) |
| 294 | self.norm1 = norm_func(self.mhsa_out_channels) |
| 295 | self.e_mhsa = E_MHSA(self.mhsa_out_channels, head_dim=head_dim, sr_ratio=sr_ratio, |
| 296 | attn_drop=attn_drop, proj_drop=drop) |
| 297 | self.mhsa_path_dropout = DropPath(path_dropout * mix_block_ratio) |
| 298 | |
| 299 | self.projection = PatchEmbed(self.mhsa_out_channels, self.mhca_out_channels, stride=1) |
| 300 | self.mhca = MHCA(self.mhca_out_channels, head_dim=head_dim) |
| 301 | self.mhca_path_dropout = DropPath(path_dropout * (1 - mix_block_ratio)) |
| 302 | |
| 303 | self.norm2 = norm_func(out_channels) |
| 304 | self.mlp = Mlp(out_channels, mlp_ratio=mlp_ratio, drop=drop) |
| 305 | self.mlp_path_dropout = DropPath(path_dropout) |
| 306 | |
| 307 | self.is_bn_merged = False |
| 308 | |
| 309 | def merge_bn(self): |
| 310 | if not self.is_bn_merged: |
| 311 | self.e_mhsa.merge_bn(self.norm1) |
| 312 | self.mlp.merge_bn(self.norm2) |
| 313 | self.is_bn_merged = True |
| 314 | |
| 315 | def forward(self, x): |
| 316 | x = self.patch_embed(x) |
| 317 | B, C, H, W = x.shape |
| 318 | if not torch.onnx.is_in_onnx_export() and not self.is_bn_merged: |
| 319 | out = self.norm1(x) |
| 320 | else: |
| 321 | out = x |
| 322 | out = rearrange(out, "b c h w -> b (h w) c") # b n c |
| 323 | out = self.mhsa_path_dropout(self.e_mhsa(out)) |
| 324 | x = x + rearrange(out, "b (h w) c -> b c h w", h=H) |
| 325 | |
| 326 | out = self.projection(x) |
| 327 | out = out + self.mhca_path_dropout(self.mhca(out)) |
| 328 | x = torch.cat([x, out], dim=1) |
| 329 | |
| 330 | if not torch.onnx.is_in_onnx_export() and not self.is_bn_merged: |
| 331 | out = self.norm2(x) |
| 332 | else: |
| 333 | out = x |