diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 1db5d2b17..0f715e26d 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -228,7 +228,7 @@ def get_response_class( if url is not None: url_parts = urlparse(url) http_origin = url_parts.scheme in ("http", "https") - if not encoding: + if not http_origin and not encoding: encoding, path_mime_type = ( _get_encoding_or_mime_type_from_path(url_parts.path) ) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 7b7819ee2..a27e2cc9f 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -28,6 +28,24 @@ __doctests__ = ['scrapy.utils.response'] # Scenarios that work the same with the previously-used, deprecated # scrapy.responsetypes.responsetypes.from_args PRE_XTRACTMIME_SCENARIOS = ( + # Content-Type determines the type for the HTTP protocol. + *( + ( + { + "url": f"{protocol}://example.com/foo", + "headers": Headers({"Content-Type": content_type}), + }, + response_class, + ) + for protocol in ("http", "https") + for content_type, response_class in ( + ("application/octet-stream", Response), + ("text/plain", TextResponse), + ("text/html", HtmlResponse), + ("text/xml", XmlResponse), + ) + ), + # Even if the body is binary, if the Content-Type says it is text, we # interpret it as text, as long as the Content-Type is not one of the 4 # affected by the Apache bug. @@ -110,19 +128,8 @@ PRE_XTRACTMIME_SCENARIOS = ( ) ), - # HTTP headers take priority over local file extensions. - ( - { - 'url': 'file.txt', - 'headers': Headers( - { - 'Content-Type': ['text/html'], - } - ), - }, - HtmlResponse, - ), - + # We continue to support the hard-coded MIME-to-Response-class mappings + # from scrapy.responsetypes. *( ( { @@ -134,12 +141,21 @@ PRE_XTRACTMIME_SCENARIOS = ( ) for mime_type, class_path in ResponseTypes.CLASSES.items() ), - ( - { - 'url': 'http://www.example.com/data.csv', - }, - TextResponse, + + # We take the file extension of URL paths into account, except for HTTP + # responses, because “they are unreliable and easily spoofed”. + # + # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata + *( + ( + {'url': f'{protocol}://example.com/a.html'}, + response_class, + ) + for protocol, response_class in ( + *((protocol, HtmlResponse) for protocol in ("file", "ftp")), + ) ), + ( { 'url': 'http://www.example.com/item/', @@ -180,7 +196,6 @@ PRE_XTRACTMIME_SCENARIOS = ( }, TextResponse, ), - ({'url': 'http://www.example.com/item/file.html'}, HtmlResponse), ({'body': b'Hello'}, HtmlResponse), ({'body': b'