diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 6d1b575b8..fd0b59de4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -61,6 +61,16 @@ _mime_overrides = get_data('scrapy', 'mime.types') or b'' _MIME_TYPES.readfp(StringIO(_mime_overrides.decode())) + +def _is_html_mime_type(mime_type): + if mime_type in { + b'application/xhtml+xml', + b'application/vnd.wap.xhtml+xml', + }: + return True + return is_html_mime_type(mime_type) + + def _is_other_text_mime_type(mime_type): return ( mime_type.startswith(b'text/') @@ -75,7 +85,7 @@ def _is_other_text_mime_type(mime_type): _PRIORITIZED_MIME_TYPE_CHECKERS = ( - is_html_mime_type, + _is_html_mime_type, is_xml_mime_type, _is_other_text_mime_type, ) @@ -123,7 +133,7 @@ def _get_mime_type_from_path(path): def _get_response_class_from_mime_type(mime_type): if not mime_type: return Response - if is_html_mime_type(mime_type): + if _is_html_mime_type(mime_type): return HtmlResponse if is_xml_mime_type(mime_type): return XmlResponse diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index f67fdb40e..bd229881f 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -6,6 +6,8 @@ import pytest from scrapy.http import HtmlResponse, Response, TextResponse, XmlResponse from scrapy.http.headers import Headers +from scrapy.responsetypes import ResponseTypes +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_meta_refresh, @@ -22,6 +24,17 @@ __doctests__ = ['scrapy.utils.response'] # Scenarios that work the same with the previously-used, deprecated # scrapy.responsetypes.responsetypes.from_args PRE_XTRACTMIME_SCENARIOS = ( + *( + ( + { + 'headers': Headers( + {'Content-Type': [mime_type]} + ), + }, + load_object(class_path), + ) + for mime_type, class_path in ResponseTypes.CLASSES.items() + ), ( { 'url': 'http://www.example.com/data.csv',