diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 35bf8baf6..62accaf17 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -86,7 +86,15 @@ def _get_encoding_or_mime_type_from_headers( encodings = headers.getlist(b'Content-Encoding') if encodings: return encodings[-1], None - if b'Content-Type' in headers: + if ( + b'Content-Type' in headers + and headers[b'Content-Type'].split(b";")[0].strip().lower() not in ( + b"", + b"unknown/unknown", + b"application/unknown", + b"*/*", + ) + ): return None, headers[b'Content-Type'] if b'Content-Disposition' in headers: path = ( diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 0ef775ff4..171ada62d 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -338,6 +338,51 @@ PRE_XTRACTMIME_SCENARIOS = ( (b"a"*RESOURCE_HEADER_BUFFER_LENGTH + BINARY_BYTES[0], TextResponse), ) ), + + # A Content-Type whose essence is "unknown/unknown", "application/unknown", + # or "*/*" has the same effect as no Content-Type being defined. + # + # https://mimesniff.spec.whatwg.org/#mime-type-sniffing-algorithm + *( + ( + { + 'body': b'