r""" Black treats `\\\n` at the end of a line as a 'NL' token, while it is ignored as whitespace in the regular Python parser. But, only the first one. If there's a `\\\n` following it (as in, a \ just by itself on a line), that is not made into NL.
(
token: pytokens.Token, source: str, prev_token: pytokens.Token | None
)
| 105 | |
| 106 | |
| 107 | def transform_whitespace( |
| 108 | token: pytokens.Token, source: str, prev_token: pytokens.Token | None |
| 109 | ) -> pytokens.Token: |
| 110 | r""" |
| 111 | Black treats `\\\n` at the end of a line as a 'NL' token, while it |
| 112 | is ignored as whitespace in the regular Python parser. |
| 113 | But, only the first one. If there's a `\\\n` following it |
| 114 | (as in, a \ just by itself on a line), that is not made into NL. |
| 115 | """ |
| 116 | if ( |
| 117 | token.type == TokenType.whitespace |
| 118 | and prev_token is not None |
| 119 | and prev_token.type not in (TokenType.nl, TokenType.newline) |
| 120 | ): |
| 121 | token_str = source[token.start_index : token.end_index] |
| 122 | if token_str.startswith("\\\r\n"): |
| 123 | return pytokens.Token( |
| 124 | TokenType.nl, |
| 125 | token.start_index, |
| 126 | token.start_index + 3, |
| 127 | token.start_line, |
| 128 | token.start_col, |
| 129 | token.start_line, |
| 130 | token.start_col + 3, |
| 131 | ) |
| 132 | elif token_str.startswith("\\\n") or token_str.startswith("\\\r"): |
| 133 | return pytokens.Token( |
| 134 | TokenType.nl, |
| 135 | token.start_index, |
| 136 | token.start_index + 2, |
| 137 | token.start_line, |
| 138 | token.start_col, |
| 139 | token.start_line, |
| 140 | token.start_col + 2, |
| 141 | ) |
| 142 | |
| 143 | return token |
| 144 | |
| 145 | |
| 146 | def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]: |