Duration predictor module. This is a module of duration predictor described in `FastSpeech: Fast, Robust and Controllable Text to Speech`_. The duration predictor predicts a duration of each frame in log domain from the hidden embeddings of encoder. .. _`FastSpeech: Fast, Robust and Cont
| 57 | |
| 58 | |
| 59 | class DurationPredictor(torch.nn.Module): |
| 60 | """Duration predictor module. |
| 61 | This is a module of duration predictor described in `FastSpeech: Fast, Robust and Controllable Text to Speech`_. |
| 62 | The duration predictor predicts a duration of each frame in log domain from the hidden embeddings of encoder. |
| 63 | .. _`FastSpeech: Fast, Robust and Controllable Text to Speech`: |
| 64 | https://arxiv.org/pdf/1905.09263.pdf |
| 65 | Note: |
| 66 | The calculation domain of outputs is different between in `forward` and in `inference`. In `forward`, |
| 67 | the outputs are calculated in log domain but in `inference`, those are calculated in linear domain. |
| 68 | """ |
| 69 | |
| 70 | def __init__(self, idim, n_layers=2, n_chans=384, kernel_size=3, dropout_rate=0.1, offset=1.0, padding='SAME'): |
| 71 | """Initilize duration predictor module. |
| 72 | Args: |
| 73 | idim (int): Input dimension. |
| 74 | n_layers (int, optional): Number of convolutional layers. |
| 75 | n_chans (int, optional): Number of channels of convolutional layers. |
| 76 | kernel_size (int, optional): Kernel size of convolutional layers. |
| 77 | dropout_rate (float, optional): Dropout rate. |
| 78 | offset (float, optional): Offset value to avoid nan in log domain. |
| 79 | """ |
| 80 | super(DurationPredictor, self).__init__() |
| 81 | self.offset = offset |
| 82 | self.conv = torch.nn.ModuleList() |
| 83 | self.kernel_size = kernel_size |
| 84 | self.padding = padding |
| 85 | for idx in range(n_layers): |
| 86 | in_chans = idim if idx == 0 else n_chans |
| 87 | self.conv += [torch.nn.Sequential( |
| 88 | torch.nn.ConstantPad1d(((kernel_size - 1) // 2, (kernel_size - 1) // 2) |
| 89 | if padding == 'SAME' |
| 90 | else (kernel_size - 1, 0), 0), |
| 91 | torch.nn.Conv1d(in_chans, n_chans, kernel_size, stride=1, padding=0), |
| 92 | torch.nn.ReLU(), |
| 93 | LayerNorm(n_chans, dim=1), |
| 94 | torch.nn.Dropout(dropout_rate) |
| 95 | )] |
| 96 | if hparams['dur_loss'] in ['mse', 'huber']: |
| 97 | odims = 1 |
| 98 | elif hparams['dur_loss'] == 'mog': |
| 99 | odims = 15 |
| 100 | elif hparams['dur_loss'] == 'crf': |
| 101 | odims = 32 |
| 102 | from torchcrf import CRF |
| 103 | self.crf = CRF(odims, batch_first=True) |
| 104 | self.linear = torch.nn.Linear(n_chans, odims) |
| 105 | |
| 106 | def _forward(self, xs, x_masks=None, is_inference=False): |
| 107 | xs = xs.transpose(1, -1) # (B, idim, Tmax) |
| 108 | for f in self.conv: |
| 109 | xs = f(xs) # (B, C, Tmax) |
| 110 | if x_masks is not None: |
| 111 | xs = xs * (1 - x_masks.float())[:, None, :] |
| 112 | |
| 113 | xs = self.linear(xs.transpose(1, -1)) # [B, T, C] |
| 114 | xs = xs * (1 - x_masks.float())[:, :, None] # (B, T, C) |
| 115 | if is_inference: |
| 116 | return self.out2dur(xs), xs |