Forward pass for training. Args: x: TODO.
(self, x: torch.Tensor)
| 103 | self.merged = True |
| 104 | |
| 105 | def forward(self, x: torch.Tensor): |
| 106 | """Forward pass for training. |
| 107 | |
| 108 | Args: |
| 109 | x: TODO. |
| 110 | """ |
| 111 | if self.r > 0 and not self.merged: |
| 112 | result = nn.Embedding.forward(self, x) |
| 113 | if self.r > 0: |
| 114 | after_A = F.embedding( |
| 115 | x, |
| 116 | self.lora_A.T, |
| 117 | self.padding_idx, |
| 118 | self.max_norm, |
| 119 | self.norm_type, |
| 120 | self.scale_grad_by_freq, |
| 121 | self.sparse, |
| 122 | ) |
| 123 | result += (after_A @ self.lora_B.T) * self.scaling |
| 124 | return result |
| 125 | else: |
| 126 | return nn.Embedding.forward(self, x) |
| 127 | |
| 128 | |
| 129 | class Linear(nn.Linear, LoRALayer): |