From 3fc9b732cb24764dbe9266a6a2e3540b3d839378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 21 Jun 2022 11:12:10 +0200 Subject: [PATCH] Refactor responsetypes --- scrapy/responsetypes.py | 21 +++++++++++++------- tests/test_downloadermiddleware_httpcache.py | 2 +- tests/test_responsetypes.py | 12 ++++++++--- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 77b190a98..13f9df52a 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -8,7 +8,11 @@ from io import StringIO from urllib.parse import urlparse from warnings import warn -from xtractmime import RESOURCE_HEADER_BUFFER_LENGTH, extract_mime, is_binary_data +from xtractmime import ( + RESOURCE_HEADER_BUFFER_LENGTH, + extract_mime, + is_binary_data, +) from xtractmime.mimegroups import ( is_html_mime_type, is_javascript_mime_type, @@ -22,13 +26,13 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode -_CONTENT_ENCODING_MAP = { +_CONTENT_ENCODING_MIME_TYPES = { 'br': b'application/brotli', 'deflate': b'application/zip', 'gzip': b'application/gzip', } -_MIMETYPES = MimeTypes() -_MIMETYPES.readfp(StringIO(get_data('scrapy', 'mime.types').decode())) +_MIME_TYPES = MimeTypes() +_MIME_TYPES.readfp(StringIO(get_data('scrapy', 'mime.types').decode())) def _is_other_text_mime_type(mime_type): @@ -94,15 +98,17 @@ def _content_type_from_metadata(*, headers=None, url_path=None, filename=None): or url_mime_type ) + def _mime_type_from_path(path): - mimetype, encoding = _MIMETYPES.guess_type(path, strict=False) - encoding_mime_type = _CONTENT_ENCODING_MAP.get(encoding, None) + mimetype, encoding = _MIME_TYPES.guess_type(path, strict=False) + encoding_mime_type = _CONTENT_ENCODING_MIME_TYPES.get(encoding, None) if encoding_mime_type: return encoding_mime_type if mimetype: return mimetype.encode() return None + def _remove_nul_byte_from_text(text): """Return the text with removed null byte (b'\x00') if there are no other binary bytes in the text, otherwise return the text as-is. @@ -118,6 +124,7 @@ def _remove_nul_byte_from_text(text): return text.replace(b'\x00', b'') + def _response_type_from_mime_type(mime_type): if not mime_type: return Response @@ -204,7 +211,7 @@ class ResponseTypes: """Return the most appropriate Response class from a file name""" warn('ResponseTypes.from_filename is deprecated, ' 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) - mimetype, encoding = _MIMETYPES.guess_type(filename) + mimetype, encoding = _MIME_TYPES.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) else: diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 7945d2a61..928c007f5 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -128,7 +128,7 @@ class DefaultStorageTest(_BaseTest): with self._storage() as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response( - 'http://www.example.com/', + 'http://www.example.com', body=b'\n.', status=202, ) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 3263dcb24..ebfe78f1e 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,7 +1,13 @@ import unittest -from scrapy.responsetypes import _MIMETYPES, responsetypes, ResponseTypes -from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers +from scrapy.http import ( + Headers, + HtmlResponse, + Response, + TextResponse, + XmlResponse, +) +from scrapy.responsetypes import _MIME_TYPES, responsetypes, ResponseTypes class PreXtractmimeResponseTypes(ResponseTypes): @@ -168,7 +174,7 @@ class ResponseTypesTest(unittest.TestCase): def test_custom_mime_types_loaded(self): """Check that mime.types files shipped with Scrapy are loaded.""" self.assertEqual( - _MIMETYPES.guess_type('x.scrapytest')[0], + _MIME_TYPES.guess_type('x.scrapytest')[0], 'x-scrapy/test', )