(sent, delimiters, max_seq_length)
| 76 | |
| 77 | |
| 78 | def split_long_sent(sent, delimiters, max_seq_length): |
| 79 | parts = [] |
| 80 | offset = 0 |
| 81 | for idx, char in enumerate(sent): |
| 82 | if char in delimiters: |
| 83 | parts.append(sent[offset:idx + 1]) |
| 84 | offset = idx + 1 |
| 85 | if not parts: |
| 86 | yield sent |
| 87 | return |
| 88 | short = [] |
| 89 | for idx, part in enumerate(parts): |
| 90 | short += part |
| 91 | if idx == len(parts) - 1: |
| 92 | yield short |
| 93 | else: |
| 94 | if len(short) + len(parts[idx + 1]) > max_seq_length: |
| 95 | yield short |
| 96 | short = [] |
| 97 | |
| 98 | |
| 99 | def possible_tokenization(text: str) -> List[Tuple[str]]: |
no test coverage detected
searching dependent graphs…