diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 62accaf17..f1efa2b73 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -83,7 +83,10 @@ def _get_encoding_or_mime_type_from_headers( headers: Headers, ) -> Tuple[Optional[bytes], Optional[bytes]]: if b'Content-Encoding' in headers: - encodings = headers.getlist(b'Content-Encoding') + encodings = [ + item.strip() for item in + b",".join(headers.getlist(b'Content-Encoding')).split(b",") + ] if encodings: return encodings[-1], None if ( diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 909e75d33..0f8e99999 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -19,6 +19,7 @@ from scrapy.utils.response import ( open_in_browser, response_httprepr, response_status_message, + _get_encoding_or_mime_type_from_headers, ) @@ -643,6 +644,26 @@ def test_get_response_class_http(kwargs, response_class): assert get_response_class(**kwargs) == response_class +@pytest.mark.parametrize( + 'headers,expected', + ( + *( + ( + Headers({'Content-Encoding': content_encoding_header}), + (encoding, None), + ) + for content_encoding_header, encoding in ( + (['gzip'], b'gzip'), + (['gzip', 'compress'], b'compress'), + (['deflate, br'], b'br'), + ) + ), + ), +) +def test_get_encoding_or_mime_type_from_headers(headers, expected): + assert _get_encoding_or_mime_type_from_headers(headers) == expected + + class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')