(self, text, pad=True)
| 173 | return tokens |
| 174 | |
| 175 | def encode(self, text, pad=True): |
| 176 | if not isinstance(text, list): |
| 177 | return self.encode([text], pad=pad) |
| 178 | |
| 179 | pad_token = self.pad_token if self.pad_token >= 0 else 0 |
| 180 | tokens = self.tokenize(text, pad=pad) |
| 181 | length = max(len(t) for t in tokens) |
| 182 | for t in tokens: |
| 183 | t.extend([pad_token] * (length - len(t))) |
| 184 | |
| 185 | return mx.array(tokens) |
no test coverage detected