Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit cb90060

Browse filesBrowse files
CalNightingalechalmerlowe
CalNightingale
and
chalmerlowe
authored
fix: add max_length support to Gzip/Brotli decoders for urllib3 2.6+ (#495)
## Summary Fixes #491. Supersedes #492 (stale). `urllib3 >= 2.6.0` now passes `max_length` to `decoder.decompress()` and accesses `decoder.has_unconsumed_tail`. Our custom `_GzipDecoder` and `_BrotliDecoder` wrappers didn't accept these, causing `TypeError` at runtime. This PR: - Updates `_GzipDecoder.decompress` and `_BrotliDecoder.decompress` to accept and forward `max_length`, with a `TypeError` fallback for older urllib3 - Adds `has_unconsumed_tail` property to `_BrotliDecoder` (the proxy class that needs it explicitly) - Applies the same fix to the async `_GzipDecoder` in `google/_async_resumable_media/` - Adds test coverage for the new `max_length` forwarding, the fallback path, and `has_unconsumed_tail` ## Feedback from #492 addressed - **`max_length=-1` default** (chandra-siri) — uses `-1` to match [urllib3's default](https://github.com/urllib3/urllib3/blob/bfe8e198a13800e3ee8ef8124a8928acb170c843/src/urllib3/response.py#L55), not `None` - **`has_unconsumed_tail` only on `_BrotliDecoder`** (chandra-siri) — `_GzipDecoder` inherits this from the parent `urllib3.response.GzipDecoder`, so no override is needed - **Lint failure** (BrennaEpp) — no trailing whitespace - **Missing test coverage for fallback paths** (BrennaEpp) — added tests for `max_length` forwarding, `TypeError` fallback, and `has_unconsumed_tail` (with `AttributeError` fallback for older urllib3) ## Test plan - [x] `pytest tests/unit/requests/test_download.py` — all pass - [x] `pytest tests_async/unit/requests/test_download.py` — all pass - [x] Full unit suite (514 tests) — all pass - [x] No lint issues (flake8 clean) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Chalmer Lowe <chalmerlowe@google.com>
1 parent 1bfd9a7 commit cb90060
Copy full SHA for cb90060

4 files changed

+113-6Lines changed: 113 additions & 6 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎packages/google-resumable-media/google/_async_resumable_media/requests/download.py‎

Copy file name to clipboardExpand all lines: packages/google-resumable-media/google/_async_resumable_media/requests/download.py
+7-2Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -452,14 +452,19 @@ def __init__(self, checksum):
452452
super(_GzipDecoder, self).__init__()
453453
self._checksum = checksum
454454

455-
def decompress(self, data):
455+
def decompress(self, data, max_length=-1):
456456
"""Decompress the bytes.
457457
458458
Args:
459459
data (bytes): The compressed bytes to be decompressed.
460+
max_length (int): Maximum number of bytes to return. -1 for no
461+
limit. Forwarded to the underlying decoder when supported.
460462
461463
Returns:
462464
bytes: The decompressed bytes from ``data``.
463465
"""
464466
self._checksum.update(data)
465-
return super(_GzipDecoder, self).decompress(data)
467+
try:
468+
return super(_GzipDecoder, self).decompress(data, max_length=max_length)
469+
except TypeError:
470+
return super(_GzipDecoder, self).decompress(data)
Collapse file

‎packages/google-resumable-media/google/resumable_media/requests/download.py‎

Copy file name to clipboardExpand all lines: packages/google-resumable-media/google/resumable_media/requests/download.py
+21-4Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -667,17 +667,22 @@ def __init__(self, checksum):
667667
super().__init__()
668668
self._checksum = checksum
669669

670-
def decompress(self, data):
670+
def decompress(self, data, max_length=-1):
671671
"""Decompress the bytes.
672672
673673
Args:
674674
data (bytes): The compressed bytes to be decompressed.
675+
max_length (int): Maximum number of bytes to return. -1 for no
676+
limit. Forwarded to the underlying decoder when supported.
675677
676678
Returns:
677679
bytes: The decompressed bytes from ``data``.
678680
"""
679681
self._checksum.update(data)
680-
return super().decompress(data)
682+
try:
683+
return super().decompress(data, max_length=max_length)
684+
except TypeError:
685+
return super().decompress(data)
681686

682687

683688
# urllib3.response.BrotliDecoder might not exist depending on whether brotli is
@@ -703,17 +708,29 @@ def __init__(self, checksum):
703708
self._decoder = urllib3.response.BrotliDecoder()
704709
self._checksum = checksum
705710

706-
def decompress(self, data):
711+
def decompress(self, data, max_length=-1):
707712
"""Decompress the bytes.
708713
709714
Args:
710715
data (bytes): The compressed bytes to be decompressed.
716+
max_length (int): Maximum number of bytes to return. -1 for no
717+
limit. Forwarded to the underlying decoder when supported.
711718
712719
Returns:
713720
bytes: The decompressed bytes from ``data``.
714721
"""
715722
self._checksum.update(data)
716-
return self._decoder.decompress(data)
723+
try:
724+
return self._decoder.decompress(data, max_length=max_length)
725+
except TypeError:
726+
return self._decoder.decompress(data)
727+
728+
@property
729+
def has_unconsumed_tail(self):
730+
try:
731+
return self._decoder.has_unconsumed_tail
732+
except AttributeError:
733+
return False
717734

718735
def flush(self):
719736
return self._decoder.flush()
Collapse file

‎packages/google-resumable-media/tests/unit/requests/test_download.py‎

Copy file name to clipboardExpand all lines: packages/google-resumable-media/tests/unit/requests/test_download.py
+70Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1274,6 +1274,37 @@ def test_decompress(self):
12741274
assert result == b""
12751275
md5_hash.update.assert_called_once_with(data)
12761276

1277+
def test_decompress_with_max_length(self):
1278+
md5_hash = mock.Mock(spec=["update"])
1279+
decoder = download_mod._GzipDecoder(md5_hash)
1280+
1281+
with mock.patch.object(
1282+
type(decoder).__bases__[0], "decompress"
1283+
) as mock_super_decompress:
1284+
mock_super_decompress.return_value = b"decompressed"
1285+
data = b"\x1f\x8b\x08\x08"
1286+
result = decoder.decompress(data, max_length=10)
1287+
1288+
assert result == b"decompressed"
1289+
md5_hash.update.assert_called_once_with(data)
1290+
mock_super_decompress.assert_called_once_with(data, max_length=10)
1291+
1292+
def test_decompress_with_max_length_fallback(self):
1293+
md5_hash = mock.Mock(spec=["update"])
1294+
decoder = download_mod._GzipDecoder(md5_hash)
1295+
1296+
with mock.patch.object(
1297+
type(decoder).__bases__[0],
1298+
"decompress",
1299+
side_effect=[TypeError, b"decompressed"],
1300+
) as mock_super_decompress:
1301+
data = b"\x1f\x8b\x08\x08"
1302+
result = decoder.decompress(data, max_length=10)
1303+
1304+
assert result == b"decompressed"
1305+
md5_hash.update.assert_called_once_with(data)
1306+
assert mock_super_decompress.call_count == 2
1307+
12771308

12781309
class Test_BrotliDecoder(object):
12791310
def test_constructor(self):
@@ -1290,6 +1321,45 @@ def test_decompress(self):
12901321
assert result == b""
12911322
md5_hash.update.assert_called_once_with(data)
12921323

1324+
def test_decompress_with_max_length(self):
1325+
md5_hash = mock.Mock(spec=["update"])
1326+
decoder = download_mod._BrotliDecoder(md5_hash)
1327+
1328+
decoder._decoder = mock.Mock(spec=["decompress"])
1329+
decoder._decoder.decompress.return_value = b"decompressed"
1330+
1331+
data = b"compressed"
1332+
result = decoder.decompress(data, max_length=10)
1333+
1334+
assert result == b"decompressed"
1335+
md5_hash.update.assert_called_once_with(data)
1336+
decoder._decoder.decompress.assert_called_once_with(data, max_length=10)
1337+
1338+
def test_decompress_with_max_length_fallback(self):
1339+
md5_hash = mock.Mock(spec=["update"])
1340+
decoder = download_mod._BrotliDecoder(md5_hash)
1341+
1342+
decoder._decoder = mock.Mock(spec=["decompress"])
1343+
decoder._decoder.decompress.side_effect = [TypeError, b"decompressed"]
1344+
1345+
data = b"compressed"
1346+
result = decoder.decompress(data, max_length=10)
1347+
1348+
assert result == b"decompressed"
1349+
md5_hash.update.assert_called_once_with(data)
1350+
assert decoder._decoder.decompress.call_count == 2
1351+
1352+
def test_has_unconsumed_tail(self):
1353+
decoder = download_mod._BrotliDecoder(mock.sentinel.md5_hash)
1354+
decoder._decoder = mock.Mock(spec=["has_unconsumed_tail"])
1355+
decoder._decoder.has_unconsumed_tail = True
1356+
assert decoder.has_unconsumed_tail is True
1357+
1358+
def test_has_unconsumed_tail_fallback(self):
1359+
decoder = download_mod._BrotliDecoder(mock.sentinel.md5_hash)
1360+
decoder._decoder = mock.Mock(spec=[])
1361+
assert decoder.has_unconsumed_tail is False
1362+
12931363

12941364
def _mock_response(status_code=http.client.OK, chunks=(), headers=None):
12951365
if headers is None:
Collapse file

‎packages/google-resumable-media/tests_async/unit/requests/test_download.py‎

Copy file name to clipboardExpand all lines: packages/google-resumable-media/tests_async/unit/requests/test_download.py
+15Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,6 +761,21 @@ def test_decompress(self):
761761
assert result == b""
762762
md5_hash.update.assert_called_once_with(data)
763763

764+
def test_decompress_with_max_length(self):
765+
md5_hash = mock.Mock(spec=["update"])
766+
decoder = download_mod._GzipDecoder(md5_hash)
767+
768+
with mock.patch.object(
769+
type(decoder).__bases__[0], "decompress"
770+
) as mock_super_decompress:
771+
mock_super_decompress.return_value = b"decompressed"
772+
data = b"\x1f\x8b\x08\x08"
773+
result = decoder.decompress(data, max_length=10)
774+
775+
assert result == b"decompressed"
776+
md5_hash.update.assert_called_once_with(data)
777+
mock_super_decompress.assert_called_once_with(data, max_length=10)
778+
764779

765780
class AsyncIter:
766781
def __init__(self, items):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.