Fix _get_encoding_or_mime_type_from_headers handling on comma-separated Content-Encoding values

This commit is contained in:
Adrián Chaves 2023-01-10 18:32:27 +01:00
parent ee6f5c56b2
commit fc2833d6bd
2 changed files with 25 additions and 1 deletions

View File

@ -83,7 +83,10 @@ def _get_encoding_or_mime_type_from_headers(
headers: Headers,
) -> Tuple[Optional[bytes], Optional[bytes]]:
if b'Content-Encoding' in headers:
encodings = headers.getlist(b'Content-Encoding')
encodings = [
item.strip() for item in
b",".join(headers.getlist(b'Content-Encoding')).split(b",")
]
if encodings:
return encodings[-1], None
if (

View File

@ -19,6 +19,7 @@ from scrapy.utils.response import (
open_in_browser,
response_httprepr,
response_status_message,
_get_encoding_or_mime_type_from_headers,
)
@ -643,6 +644,26 @@ def test_get_response_class_http(kwargs, response_class):
assert get_response_class(**kwargs) == response_class
@pytest.mark.parametrize(
'headers,expected',
(
*(
(
Headers({'Content-Encoding': content_encoding_header}),
(encoding, None),
)
for content_encoding_header, encoding in (
(['gzip'], b'gzip'),
(['gzip', 'compress'], b'compress'),
(['deflate, br'], b'br'),
)
),
),
)
def test_get_encoding_or_mime_type_from_headers(headers, expected):
assert _get_encoding_or_mime_type_from_headers(headers) == expected
class ResponseUtilsTest(unittest.TestCase):
dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')