diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index b6f7b6f68..1c4303631 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -6,9 +6,11 @@ from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO from urllib.parse import urlparse +from warnings import warn from xtractmime import extract_mime +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode @@ -43,9 +45,6 @@ class ResponseTypes: def from_mimetype(self, mimetype): """Return the most appropriate Response class for the given mimetype""" - if isinstance(mimetype, bytes): - mimetype = mimetype.decode() - if mimetype is None: return Response elif mimetype in self.classes: @@ -57,12 +56,16 @@ class ResponseTypes: def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type header """ + warn('ResponseTypes.from_content_type is deprecated, ' + 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) if content_encoding: return Response mimetype = to_unicode(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): + warn('ResponseTypes.from_content_disposition is deprecated, ' + 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) try: filename = to_unicode( content_disposition, encoding='latin-1', errors='replace' @@ -74,6 +77,8 @@ class ResponseTypes: def from_headers(self, headers): """Return the most appropriate Response class by looking at the HTTP headers""" + warn('ResponseTypes.from_headers is deprecated, ' + 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) cls = Response if b'Content-Type' in headers: cls = self.from_content_type( @@ -86,39 +91,55 @@ class ResponseTypes: def from_filename(self, filename): """Return the most appropriate Response class from a file name""" + warn('ResponseTypes.from_filename is deprecated, ' + 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) else: return Response + def from_body(self, body): + """Try to guess the appropriate response based on the body content. + This method is a bit magic and could be improved in the future, but + it's not meant to be used except for special cases where response types + cannot be guess using more straightforward methods.""" + warn('ResponseTypes.from_body is deprecated, ' + 'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning) + chunk = body[:5000] + chunk = to_bytes(chunk) + if not binary_is_text(chunk): + return self.from_mimetype('application/octet-stream') + elif b"" in chunk.lower(): + return self.from_mimetype('text/html') + elif b" {retcls} != {cls}" + def test_from_body(self): + mappings = [ + (b'\x03\x02\xdf\xdd\x23', Response), + (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse), + (b'Hello', HtmlResponse), + (b' {retcls} != {cls}" + def test_from_headers(self): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse), @@ -62,16 +73,14 @@ class ResponseTypesTest(unittest.TestCase): assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_args(self): - # TODO: add more tests that check precedence between the different arguments mappings = [ ({'url': 'http://www.example.com/data.csv'}, TextResponse), - # headers takes precedence over url - ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), - 'url': 'http://www.example.com/item/'}, HtmlResponse), + ({'headers': Headers({'Content-Type': ['text/plain; charset=utf-8']}), + 'url': 'http://www.example.com/item/'}, TextResponse), ({'body': b'Some plain text data with tabs and null bytes', 'url': 'http://www.example.com/item/', - 'headers': Headers({'Content-Type': ['text/html; charset=utf-8'], - 'X-Content-Type-Options': 'nosniff'})}, TextResponse), + 'headers': Headers({'Content-Type': ['text/html; charset=utf-8'], })}, + TextResponse), ({'body': b'\x03\x02\xdf\xdd\x23', 'url': '://www.example.com/item/', 'headers': Headers({'Content-Encoding': 'UTF-8'})}, Response), ({'body': b'Some plain text data with tabs and null bytes'}, TextResponse),