RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test_get_response_class_http: progress on documenting behavior changes

This commit is contained in:
Adrián Chaves 2022-06-30 21:28:02 +02:00
parent 0a77bfe2a0
commit 42bca211d9
2 changed files with 79 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/utils/response.py
View File

@ -51,16 +51,23 @@ from scrapy.utils.python import to_bytes, to_unicode
_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
_ENCODING_MIME_TYPES = {
_ENCODING_MIME_TYPE_MAP = {
b'br': b'application/brotli',
b'compress': b'application/x-compress',
b'deflate': b'application/zip',
b'gzip': b'application/gzip',
b'zstd': b'application/zstd',
}
_ENCODING_MIME_TYPES = {*_ENCODING_MIME_TYPE_MAP.values()}
_MIME_TYPES = MimeTypes()
_mime_overrides = get_data('scrapy', 'mime.types') or b''
_MIME_TYPES.readfp(StringIO(_mime_overrides.decode()))
def _is_compressed_mime_type(mime_type):
return mime_type in _ENCODING_MIME_TYPES
def _is_html_mime_type(mime_type):
if mime_type in {
b'application/xhtml+xml',
@ -84,6 +91,7 @@ def _is_other_text_mime_type(mime_type):
_PRIORITIZED_MIME_TYPE_CHECKERS = (
_is_compressed_mime_type,
_is_html_mime_type,
is_xml_mime_type,
_is_other_text_mime_type,
@ -108,7 +116,9 @@ def _get_encoding_or_mime_types_from_headers(
) -> Sequence[bytes]:
mime_types = []
if b'Content-Encoding' in headers:
return headers.getlist(b'Content-Encoding')[-1], None
encodings = headers.getlist(b'Content-Encoding')
if encodings:
return encodings[-1], None
if b'Content-Disposition' in headers:
path = (
headers.get(b"Content-Disposition")
@ -128,7 +138,7 @@ def _get_encoding_or_mime_types_from_headers(
def _get_mime_type_from_encoding(encoding):
return (
_ENCODING_MIME_TYPES.get(encoding, None)
_ENCODING_MIME_TYPE_MAP.get(encoding, None)
or b"application/" + encoding
)

75
tests/test_utils_response.py
View File

@ -314,15 +314,72 @@ POST_XTRACTMIME_SCENARIOS = (
)
),
(
{
'body': b'Non HTML',
'headers': Headers(
{'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
),
},
Response,
),
# Compressed content should be of type Response until uncompressed.
#
# When it comes to compression, we trust the encoding from the following
# sources, in the following order:
#
# 1, Content-Encoding HTTP header
#
# 2. File extension of the file name of the Content-Disposition HTTP
# header.
#
# 3. File extension of the file name if the resource comes through a
# file-based protocol (FTP, local file system).
#(
#{
#'url': 'file.html',
#'body': b'<!DOCTYPE html>\n<title>.</title>',
#'headers': Headers(
#{
#'Content-Disposition': [
#'attachment; filename="file.html"'
#],
#'Content-Encoding': ['zip'],
#'Content-Type': ['text/html'],
#}
#),
#},
#Response,
#),
#(
#{
#'url': 'file.html',
#'body': b'<!DOCTYPE html>\n<title>.</title>',
#'headers': Headers(
#{
#'Content-Disposition': [
#'attachment; filename="file.html.zip"'
#],
#'Content-Type': ['text/html'],
#}
#),
#},
#Response,
#),
#(
#{
#'url': 'file.html.zip',
#'body': b'<!DOCTYPE html>\n<title>.</title>',
#},
#Response,
#),
# HTTP headers take priority over local file extensions.
#(
#{
#'url': 'file.html.zip',
#'body': b'<!DOCTYPE html>\n<title>.</title>',
#'headers': Headers(
#{
#'Content-Type': ['text/html'],
#}
#),
#},
#HtmlResponse,
#),
(
{
'body': b'Some plain text',