mirror of https://github.com/scrapy/scrapy.git
get_response_class: use Response for compressed data
This commit is contained in:
Adrián Chaves
parent
b7e88f4357
commit
0a77bfe2a0
|
|
@ -51,10 +51,10 @@ from scrapy.utils.python import to_bytes, to_unicode
|
|||
|
||||
|
||||
_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
|
||||
_CONTENT_ENCODING_MIME_TYPES = {
|
||||
'br': b'application/brotli',
|
||||
'deflate': b'application/zip',
|
||||
'gzip': b'application/gzip',
|
||||
_ENCODING_MIME_TYPES = {
|
||||
b'br': b'application/brotli',
|
||||
b'compress': b'application/x-compress',
|
||||
b'deflate': b'application/zip',
|
||||
}
|
||||
_MIME_TYPES = MimeTypes()
|
||||
_mime_overrides = get_data('scrapy', 'mime.types') or b''
|
||||
|
|
@ -103,10 +103,12 @@ def _get_best_mime_type(mime_types):
|
|||
return mime_types[0]
|
||||
|
||||
|
||||
def _get_http_header_mime_types(headers: Headers) -> Sequence[bytes]:
|
||||
def _get_encoding_or_mime_types_from_headers(
|
||||
headers: Headers,
|
||||
) -> Sequence[bytes]:
|
||||
mime_types = []
|
||||
if b'Content-Type' in headers:
|
||||
mime_types.append(headers[b'Content-Type'].split(b';')[0])
|
||||
if b'Content-Encoding' in headers:
|
||||
return headers.getlist(b'Content-Encoding')[-1], None
|
||||
if b'Content-Disposition' in headers:
|
||||
path = (
|
||||
headers.get(b"Content-Disposition")
|
||||
|
|
@ -115,18 +117,29 @@ def _get_http_header_mime_types(headers: Headers) -> Sequence[bytes]:
|
|||
.strip(b"\"'")
|
||||
.decode()
|
||||
)
|
||||
mime_types.append(_get_mime_type_from_path(path))
|
||||
return mime_types
|
||||
encoding, mime_type = _get_encoding_or_mime_type_from_path(path)
|
||||
if encoding:
|
||||
return encoding, None
|
||||
mime_types.append(mime_type)
|
||||
if b'Content-Type' in headers:
|
||||
mime_types.append(headers[b'Content-Type'].split(b';')[0])
|
||||
return None, mime_types
|
||||
|
||||
|
||||
def _get_mime_type_from_path(path):
|
||||
def _get_mime_type_from_encoding(encoding):
|
||||
return (
|
||||
_ENCODING_MIME_TYPES.get(encoding, None)
|
||||
or b"application/" + encoding
|
||||
)
|
||||
|
||||
|
||||
def _get_encoding_or_mime_type_from_path(path):
|
||||
mimetype, encoding = _MIME_TYPES.guess_type(path, strict=False)
|
||||
encoding_mime_type = _CONTENT_ENCODING_MIME_TYPES.get(encoding, None)
|
||||
if encoding_mime_type:
|
||||
return encoding_mime_type
|
||||
if encoding:
|
||||
return encoding.encode(), None
|
||||
if mimetype:
|
||||
return mimetype.encode()
|
||||
return None
|
||||
return None, mimetype.encode()
|
||||
return None, None
|
||||
|
||||
|
||||
def _get_response_class_from_mime_type(mime_type):
|
||||
|
|
@ -195,16 +208,29 @@ def get_response_class(
|
|||
"""Guess the most appropriate Response class based on the given
|
||||
arguments."""
|
||||
mime_types = list(declared_mime_types or [])
|
||||
encoding = None # as in compression (e.g. gzip), not charset
|
||||
if http_headers:
|
||||
mime_types.extend(_get_http_header_mime_types(http_headers))
|
||||
encoding, header_mime_types = (
|
||||
_get_encoding_or_mime_types_from_headers(http_headers)
|
||||
)
|
||||
if not encoding:
|
||||
mime_types.extend(header_mime_types)
|
||||
if url is not None:
|
||||
url_parts = urlparse(url)
|
||||
http_origin = url_parts.scheme in ("http", "https")
|
||||
mime_types.append(_get_mime_type_from_path(url_parts.path))
|
||||
if not encoding:
|
||||
encoding, path_mime_type = (
|
||||
_get_encoding_or_mime_type_from_path(url_parts.path)
|
||||
)
|
||||
if not encoding:
|
||||
mime_types.append(path_mime_type)
|
||||
else:
|
||||
http_origin = True
|
||||
body = _remove_nul_byte_from_text((body or b'')[:BODY_LIMIT])
|
||||
if mime_types:
|
||||
if encoding:
|
||||
best_mime_type = _get_mime_type_from_encoding(encoding)
|
||||
content_types = (best_mime_type,)
|
||||
elif mime_types:
|
||||
best_mime_type = _get_best_mime_type(mime_types)
|
||||
content_types = (best_mime_type,) if best_mime_type else best_mime_type
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -54,6 +54,67 @@ PRE_XTRACTMIME_SCENARIOS = (
|
|||
'text/plain; charset=windows-1252',
|
||||
)
|
||||
),
|
||||
|
||||
# JavaScript MIME types should trigger a TextResponse.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#javascript-mime-type
|
||||
*(
|
||||
(
|
||||
{'headers': Headers({'Content-Type': [content_type]})},
|
||||
TextResponse,
|
||||
)
|
||||
for content_type in (
|
||||
'application/javascript',
|
||||
'application/x-javascript',
|
||||
'text/ecmascript',
|
||||
'text/javascript',
|
||||
'text/javascript1.0',
|
||||
'text/javascript1.1',
|
||||
'text/javascript1.2',
|
||||
'text/javascript1.3',
|
||||
'text/javascript1.4',
|
||||
'text/javascript1.5',
|
||||
'text/jscript',
|
||||
'text/livescript',
|
||||
'text/x-ecmascript',
|
||||
'text/x-javascript',
|
||||
)
|
||||
),
|
||||
|
||||
# JSON MIME types should trigger a TextResponse.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#json-mime-type
|
||||
*(
|
||||
(
|
||||
{'headers': Headers({'Content-Type': [content_type]})},
|
||||
TextResponse,
|
||||
)
|
||||
for content_type in (
|
||||
'application/json',
|
||||
'text/json',
|
||||
)
|
||||
),
|
||||
|
||||
# Compressed content should be of type Response until uncompressed.
|
||||
*(
|
||||
(
|
||||
{
|
||||
'headers': Headers(
|
||||
{
|
||||
'Content-Encoding': ['zip'],
|
||||
'Content-Type': [content_type],
|
||||
}
|
||||
)
|
||||
},
|
||||
Response,
|
||||
)
|
||||
for content_type in (
|
||||
'text/html',
|
||||
'text/xml',
|
||||
'text/plain',
|
||||
)
|
||||
),
|
||||
|
||||
*(
|
||||
(
|
||||
{
|
||||
|
|
@ -174,20 +235,6 @@ PRE_XTRACTMIME_SCENARIOS = (
|
|||
},
|
||||
HtmlResponse,
|
||||
),
|
||||
(
|
||||
{
|
||||
'url': 'http://www.example.com/item/file.xml',
|
||||
'headers': Headers(
|
||||
{
|
||||
'Content-Disposition': [
|
||||
'attachment; filename="data.xml.gz"'
|
||||
],
|
||||
'Content-Type': 'application/octet-stream',
|
||||
}
|
||||
),
|
||||
},
|
||||
XmlResponse,
|
||||
),
|
||||
({'url': 'http://www.example.com/item/file.pdf'}, Response),
|
||||
({'filename': 'file.pdf'}, Response),
|
||||
(
|
||||
|
|
@ -207,7 +254,8 @@ POST_XTRACTMIME_SCENARIOS = (
|
|||
# "text/plain; charset=iso-8859-1", or "text/plain; charset=UTF-8",
|
||||
# regardless of the actual file content.
|
||||
#
|
||||
# They should be treated as binary if their content is binary.
|
||||
# They should be treated as binary if their content is binary, and as
|
||||
# text/plain otherwise.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
|
||||
*(
|
||||
|
|
@ -226,32 +274,46 @@ POST_XTRACTMIME_SCENARIOS = (
|
|||
)
|
||||
),
|
||||
|
||||
({'filename': '/tmp/temp^'}, TextResponse),
|
||||
# If the body is empty, it contains no binary data bytes, hence body-based
|
||||
# MIME type detection must interpret the result as text.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type
|
||||
({}, TextResponse),
|
||||
({'url': '/tmp/temp^'}, TextResponse),
|
||||
|
||||
# Body-based PDF detection
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type
|
||||
({'body': b'%PDF-1.4'}, Response),
|
||||
(
|
||||
{'headers': Headers({'Content-Type': ['application/ecmascript']})},
|
||||
TextResponse,
|
||||
|
||||
# JavaScript MIME types should trigger a TextResponse.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#javascript-mime-type
|
||||
*(
|
||||
(
|
||||
{'headers': Headers({'Content-Type': [content_type]})},
|
||||
TextResponse,
|
||||
)
|
||||
for content_type in (
|
||||
'application/ecmascript',
|
||||
'application/x-ecmascript',
|
||||
)
|
||||
),
|
||||
(
|
||||
{'headers': Headers({'Content-Type': ['application/ld+json']})},
|
||||
TextResponse,
|
||||
),
|
||||
(
|
||||
{
|
||||
'headers': Headers(
|
||||
{'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
|
||||
)
|
||||
},
|
||||
HtmlResponse,
|
||||
),
|
||||
(
|
||||
{
|
||||
'headers': Headers(
|
||||
{'Content-Encoding': ['zip'], 'Content-Type': ['text/plain']}
|
||||
)
|
||||
},
|
||||
TextResponse,
|
||||
|
||||
# JSON MIME types should trigger a TextResponse.
|
||||
#
|
||||
# https://mimesniff.spec.whatwg.org/#json-mime-type
|
||||
*(
|
||||
(
|
||||
{'headers': Headers({'Content-Type': [content_type]})},
|
||||
TextResponse,
|
||||
)
|
||||
for content_type in (
|
||||
'application/foo+json',
|
||||
'application/ld+json',
|
||||
)
|
||||
),
|
||||
|
||||
(
|
||||
{
|
||||
'body': b'Non HTML',
|
||||
|
|
@ -259,7 +321,7 @@ POST_XTRACTMIME_SCENARIOS = (
|
|||
{'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
|
||||
),
|
||||
},
|
||||
HtmlResponse,
|
||||
Response,
|
||||
),
|
||||
(
|
||||
{
|
||||
|
|
@ -271,6 +333,20 @@ POST_XTRACTMIME_SCENARIOS = (
|
|||
({'body': b'\x0c\x1b'}, TextResponse),
|
||||
({'body': b'this is not <html>'}, TextResponse),
|
||||
({'body': b'this is not <?xml'}, TextResponse),
|
||||
(
|
||||
{
|
||||
'url': 'http://www.example.com/item/file.xml',
|
||||
'headers': Headers(
|
||||
{
|
||||
'Content-Disposition': [
|
||||
'attachment; filename="data.xml.gz"'
|
||||
],
|
||||
'Content-Type': 'application/octet-stream',
|
||||
}
|
||||
),
|
||||
},
|
||||
Response,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue