diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 89132c3bc..c0489dd8e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -52,30 +52,39 @@ class HttpCompressionMiddleware: def process_response(self, request, response, spider): - if request.method == 'HEAD': + if ( + request.method == 'HEAD' + or not isinstance(response, Response) + or 'Content-Encoding' not in response.headers + ): return response - if isinstance(response, Response): - content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding: - encoding = content_encoding.pop() - decoded_body = self._decode(response.body, encoding.lower()) - if self.stats: - self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - self.stats.inc_value('httpcompression/response_count', spider=spider) - respcls = get_response_class( - http_headers=response.headers, - url=response.url, - body=decoded_body, - ) - kwargs = dict(cls=respcls, body=decoded_body) - if issubclass(respcls, TextResponse): - # Force recalculating the encoding based on the new, - # decoded (uncompressed) body. - kwargs['encoding'] = None - response = response.replace(**kwargs) - if not content_encoding: - del response.headers['Content-Encoding'] - + header_list = response.headers.getlist('Content-Encoding') + encodings = [ + item.strip() for item in b",".join(header_list).split(b",") + ] + if not encodings: + return response + while encodings: + encoding = encodings.pop() + decoded_body = self._decode(response.body, encoding.lower()) + if encodings: + response.headers['Content-Encoding'] = b",".join(encodings) + else: + del response.headers['Content-Encoding'] + respcls = get_response_class( + http_headers=response.headers, + url=response.url, + body=decoded_body, + ) + kwargs = dict(cls=respcls, body=decoded_body) + if issubclass(respcls, TextResponse): + # Force recalculating the encoding based on the new, + # decoded (uncompressed) body. + kwargs['encoding'] = None + response = response.replace(**kwargs) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) return response def _decode(self, body, encoding): diff --git a/tests/sample_data/compressed/html-br-gzip.bin b/tests/sample_data/compressed/html-br-gzip.bin new file mode 100644 index 000000000..57d935008 Binary files /dev/null and b/tests/sample_data/compressed/html-br-gzip.bin differ diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e0ac80fa2..ee1f48c81 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -22,6 +22,7 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + 'br,gzip': ('html-br-gzip.bin', 'br,gzip'), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin @@ -133,6 +134,37 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual('httpcompression/response_count', 1) self.assertStatsEqual('httpcompression/response_bytes', 74837) + def test_process_response_br_gzip(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + response = self._getresponse('br,gzip') + request = response.request + self.assertEqual(response.headers['Content-Encoding'], b'br,gzip') + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert newresponse.body.startswith(b"