(source: str, grammar: Grammar | None = None)
| 144 | |
| 145 | |
| 146 | def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]: |
| 147 | lines = source.split("\n") |
| 148 | lines += [""] # For newline tokens in files that don't end in a newline |
| 149 | line, column = 1, 0 |
| 150 | |
| 151 | prev_token: pytokens.Token | None = None |
| 152 | lazy_stashed: LazyStash | None = None |
| 153 | stmt_start = True |
| 154 | |
| 155 | def emit_stashed_lazy(*, as_keyword: bool) -> Iterator[TokenInfo]: |
| 156 | nonlocal lazy_stashed |
| 157 | if lazy_stashed is None: |
| 158 | return |
| 159 | |
| 160 | stashed_token, stashed_str, stashed_line = lazy_stashed |
| 161 | yield ( |
| 162 | LAZY if as_keyword else NAME, |
| 163 | stashed_str, |
| 164 | (stashed_token.start_line, stashed_token.start_col), |
| 165 | (stashed_token.end_line, stashed_token.end_col), |
| 166 | stashed_line, |
| 167 | ) |
| 168 | lazy_stashed = None |
| 169 | |
| 170 | try: |
| 171 | for token in pytokens.tokenize(source): |
| 172 | token = transform_whitespace(token, source, prev_token) |
| 173 | |
| 174 | line, column = token.start_line, token.start_col |
| 175 | if token.type == TokenType.whitespace: |
| 176 | continue |
| 177 | |
| 178 | token_str = source[token.start_index : token.end_index] |
| 179 | |
| 180 | if token.type == TokenType.newline and token_str == "": |
| 181 | # Black doesn't yield empty newline tokens at the end of a file |
| 182 | # if there's no newline at the end of a file. |
| 183 | prev_token = token |
| 184 | continue |
| 185 | |
| 186 | source_line = lines[token.start_line - 1] |
| 187 | |
| 188 | if lazy_stashed is not None and not ( |
| 189 | token.type == TokenType.identifier and token_str in ("import", "from") |
| 190 | ): |
| 191 | yield from emit_stashed_lazy(as_keyword=False) |
| 192 | |
| 193 | if ( |
| 194 | token.type == TokenType.identifier |
| 195 | and token_str == "lazy" |
| 196 | and stmt_start |
| 197 | ): |
| 198 | lazy_stashed = (token, token_str, source_line) |
| 199 | prev_token = token |
| 200 | stmt_start = False |
| 201 | continue |
| 202 | |
| 203 | if lazy_stashed is not None: |
no test coverage detected