RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix static checks

This commit is contained in:
Akshay Sharma 2022-06-20 21:10:06 -04:00
parent b9effde875
commit 0eb1de90e2
2 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrapy/responsetypes.py
View File

@ -3,10 +3,8 @@ This module implements a class which returns the appropriate Response class
based on different criteria.
"""
from mimetypes import MimeTypes
from operator import truediv
from pkgutil import get_data
from io import StringIO
from traceback import format_exception_only
from urllib.parse import urlparse
from warnings import warn
@ -50,7 +48,7 @@ class ResponseTypes:
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
self.prioritized_mime_type_checkers = (
is_html_mime_type,
is_xml_mime_type,
@ -124,13 +122,15 @@ class ResponseTypes:
chunk = to_bytes(chunk)
if not binary_is_text(chunk):
return self.from_mimetype('application/octet-stream')
elif b"<html>" in chunk.lower():
lowercase_chunk = chunk.lower()
if b"<html>" in lowercase_chunk:
return self.from_mimetype('text/html')
elif b"<?xml" in chunk.lower():
if b"<?xml" in lowercase_chunk:
return self.from_mimetype('text/xml')
else:
return self.from_mimetype('text')
if b'<!doctype html>' in lowercase_chunk:
return self.from_mimetype('text/html')
return self.from_mimetype('text')
def _is_text_mime_type(self, mime_type):
if (
mime_type.startswith(b"text/")
@ -164,7 +164,7 @@ class ResponseTypes:
):
return TextResponse
return Response
def _guess_type(self, filename=None):
mimetype, encoding = self.mimetypes.guess_type(filename)
if encoding:
@ -181,11 +181,17 @@ class ResponseTypes:
content_type_mime_type = None
if headers and b'Content-Disposition' in headers:
content_disposition_mime_type = headers.get(b'Content-Disposition').split(b';')[-1].split(b'=')[-1].strip(b'"\'').decode()
content_disposition_mime_type = (
headers.get(b"Content-Disposition")
.split(b";")[-1]
.split(b"=")[-1]
.strip(b"\"'")
.decode()
)
content_disposition_mime_type = self._guess_type(content_disposition_mime_type)
else:
content_disposition_mime_type = None
url_mime_type = self._guess_type(url) if url else None
filename_mime_type = self._guess_type(filename) if filename else None
@ -199,7 +205,7 @@ class ResponseTypes:
for mime_type_checker in self.prioritized_mime_type_checkers:
for candidate_mime_type in candidate_mime_types:
if (
candidate_mime_type is not None
candidate_mime_type is not None
and mime_type_checker(candidate_mime_type)
):
return candidate_mime_type

4
tests/test_responsetypes.py
View File

@ -90,6 +90,8 @@ class ResponseTypesTest(unittest.TestCase):
({'body': b'<html><head><title>Hello</title></head>'}, HtmlResponse),
({'body': b'<?xml version="1.0" encoding="utf-8"'}, XmlResponse),
({'body': b'Some plain text data\1\2 with tabs and\n null bytes\0'}, Response),
# https://codersblock.com/blog/the-smallest-valid-html5-page/
({'body': b'<!DOCTYPE html>\n<title>.</title>'}, HtmlResponse),
({'body': b'\x01\x02', 'headers': Headers({'Content-Type': ['application/pdf']})}, Response),
({'headers': Headers({'Content-Type': ['application/x-json']})}, TextResponse),
({'headers': Headers({'Content-Type': ['application/x-javascript']})}, TextResponse),
@ -97,7 +99,7 @@ class ResponseTypesTest(unittest.TestCase):
({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}),
'url': 'http://www.example.com/page/'}, Response),
({'headers': Headers({'Content-Type': ['application/pdf']})}, Response),
({'url': 'http://www.example.com/page/file.html',
({'url': 'http://www.example.com/page/file.html',
'headers': Headers({'Content-Type': 'application/octet-stream'})}, HtmlResponse),
({'url': 'http://www.example.com/item/file.xml',
'headers': Headers({'Content-Type': 'application/octet-stream'})}, XmlResponse),