Read a response with the thought that reading the number of bytes larger than can fit in a 32-bit int at a time via SSL in some known cases leads to an overflow error that has to be prevented if `amt` or `self.length_remaining` indicate that a problem may hap
(
self,
amt: int | None = None,
*,
read1: bool = False,
)
| 952 | self.release_conn() |
| 953 | |
| 954 | def _fp_read( |
| 955 | self, |
| 956 | amt: int | None = None, |
| 957 | *, |
| 958 | read1: bool = False, |
| 959 | ) -> bytes: |
| 960 | """ |
| 961 | Read a response with the thought that reading the number of bytes |
| 962 | larger than can fit in a 32-bit int at a time via SSL in some |
| 963 | known cases leads to an overflow error that has to be prevented |
| 964 | if `amt` or `self.length_remaining` indicate that a problem may |
| 965 | happen. |
| 966 | |
| 967 | This happens to urllib3 injected with pyOpenSSL-backed SSL-support. |
| 968 | """ |
| 969 | assert self._fp |
| 970 | c_int_max = 2**31 - 1 |
| 971 | if ( |
| 972 | (amt and amt > c_int_max) |
| 973 | or ( |
| 974 | amt is None |
| 975 | and self.length_remaining |
| 976 | and self.length_remaining > c_int_max |
| 977 | ) |
| 978 | ) and util.IS_PYOPENSSL: |
| 979 | if read1: |
| 980 | return self._fp.read1(c_int_max) |
| 981 | buffer = io.BytesIO() |
| 982 | # Besides `max_chunk_amt` being a maximum chunk size, it |
| 983 | # affects memory overhead of reading a response by this |
| 984 | # method in CPython. |
| 985 | # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum |
| 986 | # chunk size that does not lead to an overflow error, but |
| 987 | # 256 MiB is a compromise. |
| 988 | max_chunk_amt = 2**28 |
| 989 | while amt is None or amt != 0: |
| 990 | if amt is not None: |
| 991 | chunk_amt = min(amt, max_chunk_amt) |
| 992 | amt -= chunk_amt |
| 993 | else: |
| 994 | chunk_amt = max_chunk_amt |
| 995 | data = self._fp.read(chunk_amt) |
| 996 | if not data: |
| 997 | break |
| 998 | buffer.write(data) |
| 999 | del data # to reduce peak memory usage by `max_chunk_amt`. |
| 1000 | return buffer.getvalue() |
| 1001 | elif read1: |
| 1002 | return self._fp.read1(amt) if amt is not None else self._fp.read1() |
| 1003 | else: |
| 1004 | # StringIO doesn't like amt=None |
| 1005 | return self._fp.read(amt) if amt is not None else self._fp.read() |
| 1006 | |
| 1007 | def _raw_read( |
| 1008 | self, |