Encodes each byte to an id and appends the EOS token.
| 148 | |
| 149 | |
| 150 | class ByteTextEncoderWithEos(ByteTextEncoder): |
| 151 | """Encodes each byte to an id and appends the EOS token.""" |
| 152 | |
| 153 | def encode(self, s): |
| 154 | return super(ByteTextEncoderWithEos, self).encode(s) + [EOS_ID] |
| 155 | |
| 156 | |
| 157 | class TokenTextEncoder(TextEncoder): |
nothing calls this directly
no outgoing calls
no test coverage detected