mirror of https://github.com/scrapy/scrapy.git
Deprecation Warnings
This commit is contained in:
Akshay Sharma
parent
211fc62b69
commit
763ec07b7b
|
|
@ -6,9 +6,11 @@ from mimetypes import MimeTypes
|
|||
from pkgutil import get_data
|
||||
from io import StringIO
|
||||
from urllib.parse import urlparse
|
||||
from warnings import warn
|
||||
|
||||
from xtractmime import extract_mime
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Response
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
|
||||
|
|
@ -43,9 +45,6 @@ class ResponseTypes:
|
|||
|
||||
def from_mimetype(self, mimetype):
|
||||
"""Return the most appropriate Response class for the given mimetype"""
|
||||
if isinstance(mimetype, bytes):
|
||||
mimetype = mimetype.decode()
|
||||
|
||||
if mimetype is None:
|
||||
return Response
|
||||
elif mimetype in self.classes:
|
||||
|
|
@ -57,12 +56,16 @@ class ResponseTypes:
|
|||
def from_content_type(self, content_type, content_encoding=None):
|
||||
"""Return the most appropriate Response class from an HTTP Content-Type
|
||||
header """
|
||||
warn('ResponseTypes.from_content_type is deprecated, '
|
||||
'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning)
|
||||
if content_encoding:
|
||||
return Response
|
||||
mimetype = to_unicode(content_type).split(';')[0].strip().lower()
|
||||
return self.from_mimetype(mimetype)
|
||||
|
||||
def from_content_disposition(self, content_disposition):
|
||||
warn('ResponseTypes.from_content_disposition is deprecated, '
|
||||
'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning)
|
||||
try:
|
||||
filename = to_unicode(
|
||||
content_disposition, encoding='latin-1', errors='replace'
|
||||
|
|
@ -74,6 +77,8 @@ class ResponseTypes:
|
|||
def from_headers(self, headers):
|
||||
"""Return the most appropriate Response class by looking at the HTTP
|
||||
headers"""
|
||||
warn('ResponseTypes.from_headers is deprecated, '
|
||||
'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning)
|
||||
cls = Response
|
||||
if b'Content-Type' in headers:
|
||||
cls = self.from_content_type(
|
||||
|
|
@ -86,39 +91,55 @@ class ResponseTypes:
|
|||
|
||||
def from_filename(self, filename):
|
||||
"""Return the most appropriate Response class from a file name"""
|
||||
warn('ResponseTypes.from_filename is deprecated, '
|
||||
'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning)
|
||||
mimetype, encoding = self.mimetypes.guess_type(filename)
|
||||
if mimetype and not encoding:
|
||||
return self.from_mimetype(mimetype)
|
||||
else:
|
||||
return Response
|
||||
|
||||
def from_body(self, body):
|
||||
"""Try to guess the appropriate response based on the body content.
|
||||
This method is a bit magic and could be improved in the future, but
|
||||
it's not meant to be used except for special cases where response types
|
||||
cannot be guess using more straightforward methods."""
|
||||
warn('ResponseTypes.from_body is deprecated, '
|
||||
'please use ResponseTypes.from_args instead', ScrapyDeprecationWarning)
|
||||
chunk = body[:5000]
|
||||
chunk = to_bytes(chunk)
|
||||
if not binary_is_text(chunk):
|
||||
return self.from_mimetype('application/octet-stream')
|
||||
elif b"<html>" in chunk.lower():
|
||||
return self.from_mimetype('text/html')
|
||||
elif b"<?xml" in chunk.lower():
|
||||
return self.from_mimetype('text/xml')
|
||||
else:
|
||||
return self.from_mimetype('text')
|
||||
|
||||
def from_args(self, headers=None, url=None, filename=None, body=None):
|
||||
"""Guess the most appropriate Response class based on
|
||||
the given arguments."""
|
||||
if filename:
|
||||
warn("'filename' keyword argument is deprecated, "
|
||||
"please ignore this parameter as it is no longer required",
|
||||
ScrapyDeprecationWarning)
|
||||
if not body:
|
||||
body = b''
|
||||
|
||||
cls = Response
|
||||
if cls is Response and body is not None:
|
||||
http_origin = True
|
||||
no_sniff = False
|
||||
content_types = None
|
||||
http_origin = True
|
||||
content_types = None
|
||||
|
||||
if url and urlparse(url).scheme not in ("http", "https"):
|
||||
http_origin = False
|
||||
if url and urlparse(url).scheme not in ("http", "https"):
|
||||
http_origin = False
|
||||
|
||||
if headers:
|
||||
if b'Content-Type' in headers:
|
||||
content_types = (headers[b'Content-Type'],)
|
||||
if headers and b'Content-Type' in headers:
|
||||
content_types = tuple(headers.getlist(b'Content-Type'))
|
||||
|
||||
if b'X-Content-Type-Options' in headers and headers[b'X-Content-Type-Options'] == b"nosniff":
|
||||
no_sniff = True
|
||||
mime_type = extract_mime(body, content_types=content_types, http_origin=http_origin)
|
||||
cls = self.from_mimetype(mime_type.decode())
|
||||
|
||||
mime_type = extract_mime(body, content_types=content_types, http_origin=http_origin, no_sniff=no_sniff)
|
||||
cls = self.from_mimetype(mime_type)
|
||||
if cls is Response and headers is not None:
|
||||
cls = self.from_headers(headers)
|
||||
if cls is Response and url is not None:
|
||||
cls = self.from_filename(url)
|
||||
if cls is Response and filename is not None:
|
||||
cls = self.from_filename(filename)
|
||||
return cls
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,17 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
retcls = responsetypes.from_content_type(source)
|
||||
assert retcls is cls, f"{source} ==> {retcls} != {cls}"
|
||||
|
||||
def test_from_body(self):
|
||||
mappings = [
|
||||
(b'\x03\x02\xdf\xdd\x23', Response),
|
||||
(b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
|
||||
(b'<html><head><title>Hello</title></head>', HtmlResponse),
|
||||
(b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
|
||||
]
|
||||
for source, cls in mappings:
|
||||
retcls = responsetypes.from_body(source)
|
||||
assert retcls is cls, f"{source} ==> {retcls} != {cls}"
|
||||
|
||||
def test_from_headers(self):
|
||||
mappings = [
|
||||
({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
|
||||
|
|
@ -62,16 +73,14 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
assert retcls is cls, f"{source} ==> {retcls} != {cls}"
|
||||
|
||||
def test_from_args(self):
|
||||
# TODO: add more tests that check precedence between the different arguments
|
||||
mappings = [
|
||||
({'url': 'http://www.example.com/data.csv'}, TextResponse),
|
||||
# headers takes precedence over url
|
||||
({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
|
||||
'url': 'http://www.example.com/item/'}, HtmlResponse),
|
||||
({'headers': Headers({'Content-Type': ['text/plain; charset=utf-8']}),
|
||||
'url': 'http://www.example.com/item/'}, TextResponse),
|
||||
({'body': b'Some plain text data with tabs and null bytes',
|
||||
'url': 'http://www.example.com/item/',
|
||||
'headers': Headers({'Content-Type': ['text/html; charset=utf-8'],
|
||||
'X-Content-Type-Options': 'nosniff'})}, TextResponse),
|
||||
'headers': Headers({'Content-Type': ['text/html; charset=utf-8'], })},
|
||||
TextResponse),
|
||||
({'body': b'\x03\x02\xdf\xdd\x23', 'url': '://www.example.com/item/',
|
||||
'headers': Headers({'Content-Encoding': 'UTF-8'})}, Response),
|
||||
({'body': b'Some plain text data with tabs and null bytes'}, TextResponse),
|
||||
|
|
|
|||
Loading…
Reference in New Issue