Skip to content

Commit

Permalink
Rework stream to use a conventional read loop
Browse files Browse the repository at this point in the history
  • Loading branch information
smason committed Nov 30, 2023
1 parent 4ae4b71 commit f6fa97b
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
2 changes: 1 addition & 1 deletion changelog/3216.feature.rst
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
HTTPResponse.stream(amt=None) now yields data from non-chunked responses as it
is available. Previously all intermediate data was buffered and only returned
as a single bytes object at the end of the the response.
as a single, large, bytes object at the end of the the response.
14 changes: 10 additions & 4 deletions src/urllib3/response.py
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,8 @@ def _fp_read(
del data # to reduce peak memory usage by `max_chunk_amt`.
return buffer.getvalue()
elif read1:
# work around: https://github.com/python/cpython/issues/112064
amt = amt or self.length_remaining
return self._fp.read1(amt) if amt is not None else self._fp.read1()
else:
# StringIO doesn't like amt=None
Expand Down Expand Up @@ -1019,14 +1021,18 @@ def stream(
if self.chunked and self.supports_chunked_reads():
yield from self.read_chunked(amt, decode_content=decode_content)
else:
while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0:
# TODO: is silently ignoring this the right thing to do?
if is_fp_closed(self._fp) and len(self._decoded_buffer) == 0:
return
while True:
if amt is None:
data = self.read1(amt=amt, decode_content=decode_content)
else:
data = self.read(amt=amt, decode_content=decode_content)

if data:
yield data
if not data:
break
yield data
assert is_fp_closed(self._fp) and len(self._decoded_buffer) == 0

# Overrides from io.IOBase
def readable(self) -> bool:
Expand Down

0 comments on commit f6fa97b

Please sign in to comment.