diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 825272274..bf479eeb3 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -51,16 +51,23 @@ from scrapy.utils.python import to_bytes, to_unicode
_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
-_ENCODING_MIME_TYPES = {
+_ENCODING_MIME_TYPE_MAP = {
b'br': b'application/brotli',
b'compress': b'application/x-compress',
b'deflate': b'application/zip',
+ b'gzip': b'application/gzip',
+ b'zstd': b'application/zstd',
}
+_ENCODING_MIME_TYPES = {*_ENCODING_MIME_TYPE_MAP.values()}
_MIME_TYPES = MimeTypes()
_mime_overrides = get_data('scrapy', 'mime.types') or b''
_MIME_TYPES.readfp(StringIO(_mime_overrides.decode()))
+def _is_compressed_mime_type(mime_type):
+ return mime_type in _ENCODING_MIME_TYPES
+
+
def _is_html_mime_type(mime_type):
if mime_type in {
b'application/xhtml+xml',
@@ -84,6 +91,7 @@ def _is_other_text_mime_type(mime_type):
_PRIORITIZED_MIME_TYPE_CHECKERS = (
+ _is_compressed_mime_type,
_is_html_mime_type,
is_xml_mime_type,
_is_other_text_mime_type,
@@ -108,7 +116,9 @@ def _get_encoding_or_mime_types_from_headers(
) -> Sequence[bytes]:
mime_types = []
if b'Content-Encoding' in headers:
- return headers.getlist(b'Content-Encoding')[-1], None
+ encodings = headers.getlist(b'Content-Encoding')
+ if encodings:
+ return encodings[-1], None
if b'Content-Disposition' in headers:
path = (
headers.get(b"Content-Disposition")
@@ -128,7 +138,7 @@ def _get_encoding_or_mime_types_from_headers(
def _get_mime_type_from_encoding(encoding):
return (
- _ENCODING_MIME_TYPES.get(encoding, None)
+ _ENCODING_MIME_TYPE_MAP.get(encoding, None)
or b"application/" + encoding
)
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 91a1e5457..81283112e 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -314,15 +314,72 @@ POST_XTRACTMIME_SCENARIOS = (
)
),
- (
- {
- 'body': b'Non HTML',
- 'headers': Headers(
- {'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
- ),
- },
- Response,
- ),
+
+ # Compressed content should be of type Response until uncompressed.
+ #
+ # When it comes to compression, we trust the encoding from the following
+ # sources, in the following order:
+ #
+ # 1, Content-Encoding HTTP header
+ #
+ # 2. File extension of the file name of the Content-Disposition HTTP
+ # header.
+ #
+ # 3. File extension of the file name if the resource comes through a
+ # file-based protocol (FTP, local file system).
+ #(
+ #{
+ #'url': 'file.html',
+ #'body': b'\n.',
+ #'headers': Headers(
+ #{
+ #'Content-Disposition': [
+ #'attachment; filename="file.html"'
+ #],
+ #'Content-Encoding': ['zip'],
+ #'Content-Type': ['text/html'],
+ #}
+ #),
+ #},
+ #Response,
+ #),
+ #(
+ #{
+ #'url': 'file.html',
+ #'body': b'\n.',
+ #'headers': Headers(
+ #{
+ #'Content-Disposition': [
+ #'attachment; filename="file.html.zip"'
+ #],
+ #'Content-Type': ['text/html'],
+ #}
+ #),
+ #},
+ #Response,
+ #),
+ #(
+ #{
+ #'url': 'file.html.zip',
+ #'body': b'\n.',
+ #},
+ #Response,
+ #),
+
+ # HTTP headers take priority over local file extensions.
+ #(
+ #{
+ #'url': 'file.html.zip',
+ #'body': b'\n.',
+ #'headers': Headers(
+ #{
+ #'Content-Type': ['text/html'],
+ #}
+ #),
+ #},
+ #HtmlResponse,
+ #),
+
(
{
'body': b'Some plain text',