From 13bc1499ac2191d39bf9643e6c3b3770a2207240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 9 Feb 2024 08:20:49 +0100 Subject: [PATCH] Ignore Content-Type when it triggers Response --- scrapy/utils/response.py | 11 +++ tests/test_responsetypes.py | 4 +- tests/test_utils_response.py | 135 ++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 42 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 3459b47bc..2ab167478 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -203,6 +203,17 @@ def get_response_class( content_types=content_types, http_origin=http_origin, ) + cls = _get_response_class_from_mime_type(mime_type) + if cls is not Response or not content_types or encoding or not http_origin: + return cls + # In scenarios where there was a declared Content-Type, no + # Content-Encoding, HTTP/HTTPS was used, and xtractmime determined the + # output to be binary, repeat MIME extraction ignoring the declared + # Content-Type, so that the body is taken into account. + mime_type = extract_mime( + body, + http_origin=http_origin, + ) return _get_response_class_from_mime_type(mime_type) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 7d769cfda..a19a2133f 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -45,7 +45,9 @@ def _unmark(item): ), ) def test_from_args(kwargs, response_class): - assert responsetypes.from_args(**kwargs) == response_class + assert ( + responsetypes.from_args(**kwargs) == response_class + ), f"{responsetypes.from_args(**kwargs)=} != {response_class=}" class ResponseTypesTest(unittest.TestCase): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 838169e4f..f226d6c88 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -89,7 +89,6 @@ PRE_XTRACTMIME_SCENARIOS = ( # Make sure that MIME parameters do not break response class choice. for content_type_parameters in ("", "; foo=bar") for content_type, response_class in ( - ("application/octet-stream", Response), ("text/plain", TextResponse), ("text/html", HtmlResponse), ("text/xml", XmlResponse), @@ -140,10 +139,6 @@ PRE_XTRACTMIME_SCENARIOS = ( "application/x-json", ) ), - # Binary MIME types should trigger a Response. - # - # https://mimesniff.spec.whatwg.org/#json-mime-type - *((mime_type, Response) for mime_type in ("application/pdf",)), ) ), # Content-Type triumphs body, except for: @@ -174,6 +169,28 @@ PRE_XTRACTMIME_SCENARIOS = ( ), ) ), + # Content-Type triumphs Content-Disposition. + *( + ( + { + "url": f"{protocol}://example.com/a", + "headers": Headers( + { + "Content-Disposition": [ + f'attachment; filename="a.{file_extension}"', + ], + "Content-Type": [content_type], + } + ), + }, + response_class, + ) + for protocol in ("http", "https") + for file_extension, content_type, response_class in ( + ("html", "application/json", JsonResponse), + ("xml", "application/json", JsonResponse), + ) + ), # Compressed content should be of type Response until uncompressed. ( { @@ -378,6 +395,36 @@ PRE_XTRACTMIME_SCENARIOS = ( "*/*", ) ), + # Content triumphs Content-Type when using HTTP or HTTPS and the + # Content-Type is unknown or binary while the content is plain text. This + # is a conscious divergence from the MIME Sniffing Standard for a better + # web scraping experience. + *( + ( + { + "url": f"{protocol}://example.com/foo", + "headers": Headers({"Content-Type": content_type}), + "body": body, + }, + TextResponse, + ) + for protocol in ("http", "https") + for body in ( + b"", + b"a", + b"var a = 'b';", + b'{"a": "b"}', + b'.a {b: "c"}', + ) + for content_type in ( + "application/octet-stream", + "application/pdf", + "application/custom", + "application/bad-custom-json", # Should end in +json + "application/bad-custom-text", # Should start with text/ + "application/bad-custom-xml", # Should end in +xml + ) + ), ) # Scenarios that work differently with the previously-used, deprecated @@ -444,7 +491,6 @@ POST_XTRACTMIME_SCENARIOS = ( response_class, ) for body, content_type, response_class in ( - (b"a", "application/octet-stream", Response), *( (b"\x00\x01\xff", content_type, Response) for content_type in ( @@ -474,20 +520,6 @@ POST_XTRACTMIME_SCENARIOS = ( ), ) ), - # Content-Type also triumphs Content-Disposition. - ( - { - "headers": Headers( - { - "Content-Disposition": [ - 'attachment; filename="a.html"', - ], - "Content-Type": ["application/octet-stream"], - } - ) - }, - Response, - ), # Compressed content should be of type Response until uncompressed. ( { @@ -543,27 +575,6 @@ POST_XTRACTMIME_SCENARIOS = ( *((protocol, TextResponse) for protocol in ("http", "https")), ) ), - # Content-Type triumphes Content-Disposition. - *( - ( - { - "url": f"{protocol}://example.com/a", - "headers": Headers( - { - "Content-Disposition": [ - f'attachment; filename="a.{file_extension}"', - ], - "Content-Type": [content_type], - } - ), - }, - response_class, - ) - for protocol in ("http", "https") - for file_extension, content_type, response_class in ( - ("xml", "application/octet-stream", Response), - ) - ), # File extension triumphs body. ( { @@ -645,6 +656,48 @@ POST_XTRACTMIME_SCENARIOS = ( (b"a