Frontend + Encoder. Note that this method is used by asr_inference.py Args: speech: (Batch, Length, ...) speech_lengths: (Batch, )
(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
)
| 653 | return loss, stats, weight |
| 654 | |
| 655 | def encode( |
| 656 | self, speech: torch.Tensor, speech_lengths: torch.Tensor |
| 657 | ) -> Tuple[torch.Tensor, torch.Tensor]: |
| 658 | """Frontend + Encoder. Note that this method is used by asr_inference.py |
| 659 | Args: |
| 660 | speech: (Batch, Length, ...) |
| 661 | speech_lengths: (Batch, ) |
| 662 | """ |
| 663 | with autocast(False): |
| 664 | |
| 665 | # Data augmentation |
| 666 | if self.specaug is not None and self.training: |
| 667 | speech = speech.permute(0, 2, 1) |
| 668 | # suit for whisper padding |
| 669 | padded_speech_lengths = torch.ones_like(speech_lengths) * speech.shape[1] |
| 670 | speech, padded_speech_lengths = self.specaug(speech, padded_speech_lengths) |
| 671 | speech = speech.permute(0, 2, 1) |
| 672 | |
| 673 | # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN |
| 674 | if self.normalize is not None: |
| 675 | speech, speech_lengths = self.normalize(speech, speech_lengths) |
| 676 | |
| 677 | # Forward encoder |
| 678 | # feats: (Batch, Length, Dim) |
| 679 | # -> encoder_out: (Batch, Length2, Dim2) |
| 680 | if self.encoder.interctc_use_conditioning: |
| 681 | encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths, ctc=self.ctc) |
| 682 | else: |
| 683 | encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) |
| 684 | intermediate_outs = None |
| 685 | if isinstance(encoder_out, tuple): |
| 686 | intermediate_outs = encoder_out[1] |
| 687 | encoder_out = encoder_out[0] |
| 688 | |
| 689 | if intermediate_outs is not None: |
| 690 | return (encoder_out, intermediate_outs), encoder_out_lens |
| 691 | |
| 692 | return encoder_out, encoder_out_lens |
| 693 | |
| 694 | def inference( |
| 695 | self, |