From 0eb1de90e27af37833ca87b5d5cf2fcce40585df Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+akshaysharmajs@users.noreply.github.com> Date: Mon, 20 Jun 2022 21:10:06 -0400 Subject: [PATCH] fix static checks --- scrapy/responsetypes.py | 30 ++++++++++++++++++------------ tests/test_responsetypes.py | 4 +++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 816b1f040..4140d9de0 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -3,10 +3,8 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ from mimetypes import MimeTypes -from operator import truediv from pkgutil import get_data from io import StringIO -from traceback import format_exception_only from urllib.parse import urlparse from warnings import warn @@ -50,7 +48,7 @@ class ResponseTypes: self.mimetypes.readfp(StringIO(mimedata)) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - + self.prioritized_mime_type_checkers = ( is_html_mime_type, is_xml_mime_type, @@ -124,13 +122,15 @@ class ResponseTypes: chunk = to_bytes(chunk) if not binary_is_text(chunk): return self.from_mimetype('application/octet-stream') - elif b"" in chunk.lower(): + lowercase_chunk = chunk.lower() + if b"" in lowercase_chunk: return self.from_mimetype('text/html') - elif b"' in lowercase_chunk: + return self.from_mimetype('text/html') + return self.from_mimetype('text') + def _is_text_mime_type(self, mime_type): if ( mime_type.startswith(b"text/") @@ -164,7 +164,7 @@ class ResponseTypes: ): return TextResponse return Response - + def _guess_type(self, filename=None): mimetype, encoding = self.mimetypes.guess_type(filename) if encoding: @@ -181,11 +181,17 @@ class ResponseTypes: content_type_mime_type = None if headers and b'Content-Disposition' in headers: - content_disposition_mime_type = headers.get(b'Content-Disposition').split(b';')[-1].split(b'=')[-1].strip(b'"\'').decode() + content_disposition_mime_type = ( + headers.get(b"Content-Disposition") + .split(b";")[-1] + .split(b"=")[-1] + .strip(b"\"'") + .decode() + ) content_disposition_mime_type = self._guess_type(content_disposition_mime_type) else: content_disposition_mime_type = None - + url_mime_type = self._guess_type(url) if url else None filename_mime_type = self._guess_type(filename) if filename else None @@ -199,7 +205,7 @@ class ResponseTypes: for mime_type_checker in self.prioritized_mime_type_checkers: for candidate_mime_type in candidate_mime_types: if ( - candidate_mime_type is not None + candidate_mime_type is not None and mime_type_checker(candidate_mime_type) ): return candidate_mime_type diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index c309b75f0..e891a1c24 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -90,6 +90,8 @@ class ResponseTypesTest(unittest.TestCase): ({'body': b'Hello'}, HtmlResponse), ({'body': b'\n.'}, HtmlResponse), ({'body': b'\x01\x02', 'headers': Headers({'Content-Type': ['application/pdf']})}, Response), ({'headers': Headers({'Content-Type': ['application/x-json']})}, TextResponse), ({'headers': Headers({'Content-Type': ['application/x-javascript']})}, TextResponse), @@ -97,7 +99,7 @@ class ResponseTypesTest(unittest.TestCase): ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response), ({'headers': Headers({'Content-Type': ['application/pdf']})}, Response), - ({'url': 'http://www.example.com/page/file.html', + ({'url': 'http://www.example.com/page/file.html', 'headers': Headers({'Content-Type': 'application/octet-stream'})}, HtmlResponse), ({'url': 'http://www.example.com/item/file.xml', 'headers': Headers({'Content-Type': 'application/octet-stream'})}, XmlResponse),