RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More tests

This commit is contained in:
Akshay Sharma 2021-08-11 17:14:38 +05:30
parent 9980a3d134
commit 4169fd184f
2 changed files with 42 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
scrapy/responsetypes.py
View File

@ -149,36 +149,45 @@ class ResponseTypes:
if is_xml_mime_type(mime_type):
return XmlResponse
if (
mime_type.startswith(b'text/')
mime_type.startswith(b"text/")
or is_json_mime_type(mime_type)
or is_javascript_mime_type(mime_type)
or mime_type
in (
b"application/x-json",
b"application/json-amazonui-streaming",
b"application/x-javascript",
)
):
return TextResponse
return Response
def from_args(self, headers=None, url=None, filename=None, body=None):
"""Guess the most appropriate Response class based on
the given arguments."""
if not body:
body = b''
def _remove_nul_byte_from_text(self, text):
"""Return the text with removed null byte (b"\x00") if there are no other
binary bytes in the text, otherwise return the text as-is.
body = body[:RESOURCE_HEADER_BUFFER_LENGTH]
Based on https://github.com/scrapy/scrapy/issues/2481"""
contains_binary_bytes = False
for index in range(len(body)):
if body[index:index + 1] != b"\x00" and is_binary_data(body[index:index + 1]):
for index in range(len(text)):
if text[index:index + 1] != b"\x00" and is_binary_data(text[index:index + 1]):
contains_binary_bytes = True
break
if not contains_binary_bytes:
body = body.replace(b"\x00", b"")
text = text.replace(b"\x00", b"")
cls = Response
return text
def from_args(self, headers=None, url=None, filename=None, body=None):
"""Guess the most appropriate Response class based on
the given arguments."""
body = body or b''
body = self._remove_nul_byte_from_text(body[:RESOURCE_HEADER_BUFFER_LENGTH])
http_origin = not url or urlparse(url).scheme in ("http", "https")
content_types = self._guess_content_type(headers=headers, url=url, filename=filename)
mime_type = extract_mime(body, content_types=content_types, http_origin=http_origin)
cls = self._guess_response_type(mime_type)
return cls
return self._guess_response_type(mime_type)
responsetypes = ResponseTypes()

26
tests/test_responsetypes.py
View File

@ -83,14 +83,14 @@ class ResponseTypesTest(unittest.TestCase):
({'body': b'\x03\x02\xdf\xdd\x23', 'headers': Headers({'Content-Encoding': 'UTF-8'})},
Response),
({'body': b'\x00\x01\xff', 'url': '://www.example.com/item/',
'headers': Headers({'Content-Type': b'text/plain'})}, TextResponse),
'headers': Headers({'Content-Type': ['text/plain']})}, TextResponse),
({'url': 'http://www.example.com/item/file.html'}, HtmlResponse),
({'body': b'<html><head><title>Hello</title></head>'}, HtmlResponse),
({'body': b'<?xml version="1.0" encoding="utf-8"'}, XmlResponse),
({'filename': 'file.pdf'}, Response),
({'url': 'http://www.example.com/item/file.pdf'}, Response),
({'body': b'Some plain text data\1\2 with tabs and\n null bytes\0'}, Response),
({'filename': '/tmp/temp^'}, TextResponse),
({'headers': Headers({'Content-Type': ['application/x-json']})}, TextResponse),
]
for source, cls in mappings:
retcls = responsetypes.from_args(**source)
@ -98,13 +98,25 @@ class ResponseTypesTest(unittest.TestCase):
def test_from_args_post_xtractmime(self):
mappings = [
# different behaviour with http and non-http urls
({'body': b'\x00\x01\xff', 'url': 'http://www.example.com/item/',
'headers': Headers({'Content-Type': b'text/plain'})}, Response),
'headers': Headers({'Content-Type': ['text/plain']})}, Response),
({'filename': '/tmp/temp^'}, TextResponse),
({'body': b'%PDF-1.4'}, Response),
({'body': b'%PDF-1.4', 'headers': Headers({'Content-Type': b'application/pdf'})}, Response),
({'headers': Headers({'Content-Type': b'application/ecmascript'})}, TextResponse),
({'headers': Headers({'Content-Type': b'application/ld+json'})}, TextResponse),
({'headers': Headers({'Content-Type': ['application/pdf']})}, Response),
({'headers': Headers({'Content-Type': ['application/ecmascript']})}, TextResponse),
({'headers': Headers({'Content-Type': ['application/ld+json']})}, TextResponse),
({'headers': Headers({'Content-Type': ['application/x-javascript']})}, TextResponse),
({'headers': Headers({'Content-Encoding': ['zip'], 'Content-Type': ['text/html']})},
HtmlResponse),
({'headers': Headers({'Content-Encoding': ['zip'], 'Content-Type': ['text/plain']})},
TextResponse),
({'body': b'Non HTML', 'headers': Headers({'Content-Encoding': ['zip'],
'Content-Type': ['text/html']})}, HtmlResponse),
({'body': b'Some plain text', 'headers': Headers({'Content-Type': 'application/octet-stream'})},
Response),
({'body': b'\x0c\x1b'}, TextResponse),
({'body': b'this is not <html>'}, TextResponse),
({'body': b'this is not <?xml'}, TextResponse),
]
for source, cls in mappings:
retcls = responsetypes.from_args(**source)