diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 709333948..1e340abb6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, List, Optional, Union @@ -102,18 +103,18 @@ class HttpCompressionMiddleware: if isinstance(response, Response): content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: - encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) warn_size = request.meta.get("download_warnsize", self._warn_size) try: - decoded_body = self._decode( - response.body, encoding.lower(), max_size + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({max_size} B) during decompression." + f"({len(response.body)} B compressed) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during " + f"decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( @@ -121,6 +122,7 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -144,6 +146,28 @@ class HttpCompressionMiddleware: return response + def _handle_encoding(self, body, content_encoding, max_size): + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + + def _split_encodings(self, content_encoding): + to_keep = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in ACCEPTED_ENCODINGS: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": return gunzip(body, max_size=max_size) diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 000000000..d66f4c5a0 Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin differ diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 000000000..5066842ef Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate.bin differ diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9deb81c37..ae5569d0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -27,6 +27,8 @@ FORMAT = { "x-gzip": ("html-gzip.bin", "gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), @@ -205,6 +207,62 @@ class HttpCompressionTest(TestCase): assert newresponse is not response self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"]) + def test_multi_compression_single_header(self): + response = self._getresponse("gzip-deflate") + request = response.request + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert "Content-Encoding" not in newresponse.headers + assert newresponse.body.startswith(b"