(token)
| 10 | |
| 11 | |
| 12 | def ispunct(token): |
| 13 | return all(unicodedata.category(char).startswith('P') |
| 14 | for char in token) |
| 15 | |
| 16 | |
| 17 | def split_long_sentence_into(tokens: List[str], max_seq_length, sent_delimiter=None, char_level=False, |
no outgoing calls
no test coverage detected
searching dependent graphs…