Converts a bytes string with python source code to unicode. Unicode strings are passed through unchanged. Byte strings are checked for the python source file encoding cookie to determine encoding. txt can be either a bytes buffer or a string containing the source code.
(txt, errors='replace', skip_encoding_cookie=True)
| 14 | cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE) |
| 15 | |
| 16 | def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): |
| 17 | """Converts a bytes string with python source code to unicode. |
| 18 | |
| 19 | Unicode strings are passed through unchanged. Byte strings are checked |
| 20 | for the python source file encoding cookie to determine encoding. |
| 21 | txt can be either a bytes buffer or a string containing the source |
| 22 | code. |
| 23 | """ |
| 24 | if isinstance(txt, str): |
| 25 | return txt |
| 26 | if isinstance(txt, bytes): |
| 27 | buffer = BytesIO(txt) |
| 28 | else: |
| 29 | buffer = txt |
| 30 | try: |
| 31 | encoding, _ = detect_encoding(buffer.readline) |
| 32 | except SyntaxError: |
| 33 | encoding = "ascii" |
| 34 | buffer.seek(0) |
| 35 | with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text: |
| 36 | text.mode = 'r' |
| 37 | if skip_encoding_cookie: |
| 38 | return u"".join(strip_encoding_cookie(text)) |
| 39 | else: |
| 40 | return text.read() |
| 41 | |
| 42 | def strip_encoding_cookie(filelike): |
| 43 | """Generator to pull lines from a text-mode file, skipping the encoding |
no test coverage detected