RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some simplifications to Request and Response classes

This commit is contained in:
Pablo Hoffman 2009-08-24 09:47:26 -03:00
parent 81832773ee
commit b71de57b21
6 changed files with 31 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/http/request/form.py
View File

@ -26,7 +26,7 @@ class FormRequest(Request):
def __init__(self, *args, **kwargs):
formdata = kwargs.pop('formdata', None)
Request.__init__(self, *args, **kwargs)
super(FormRequest, self).__init__(*args, **kwargs)
if formdata:
items = formdata.iteritems() if isinstance(formdata, dict) else formdata

4
scrapy/http/request/rpc.py
View File

@ -12,6 +12,8 @@ from scrapy.http.request import Request
class XmlRpcRequest(Request):
__slots__ = ()
def __init__(self, *args, **kwargs):
if 'body' not in kwargs:
params = kwargs.pop('params')
@ -24,5 +26,5 @@ class XmlRpcRequest(Request):
# xmlrpc query multiples times over the same url
kwargs.setdefault('dont_filter', True)
Request.__init__(self, *args, **kwargs)
super(XmlRpcRequest, self).__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'text/xml')

12
scrapy/http/response/__init__.py
View File

@ -44,17 +44,20 @@ class Response(object_ref):
if isinstance(body, str):
self._body = body
elif isinstance(body, unicode):
raise TypeError("Cannot assign a unicode body to a raw Response. Use TextResponse, HtmlResponse, etc")
raise TypeError("Cannot assign a unicode body to a raw Response. " \
"Use TextResponse, HtmlResponse, etc")
elif body is None:
self._body = ''
else:
raise TypeError("Response body must either str or unicode. Got: '%s'" % type(body).__name__)
raise TypeError("Response body must either str or unicode. Got: '%s'" \
% type(body).__name__)
body = property(_get_body, _set_body)
def __repr__(self):
return "%s(url=%s, headers=%s, status=%s, body=%s)" % \
(type(self).__name__, repr(self.url), repr(self.headers), repr(self.status), repr(self.body))
(type(self).__name__, repr(self.url), repr(self.headers), \
repr(self.status), repr(self.body))
def __str__(self):
flags = "(%s) " % ",".join(self.flags) if self.flags else ""
@ -65,7 +68,8 @@ class Response(object_ref):
"""Return a copy of this Response"""
return self.replace()
def replace(self, url=None, status=None, headers=None, body=None, meta=None, flags=None, cls=None, **kwargs):
def replace(self, url=None, status=None, headers=None, body=None, meta=None, \
flags=None, cls=None, **kwargs):
"""Create a new Response with the same attributes except for those
given new values.
"""

2
scrapy/http/response/html.py
View File

@ -24,7 +24,7 @@ class HtmlResponse(TextResponse):
METATAG_RE2 = re.compile(r'<meta\s+%s\s+%s' % (_content_re, _httpequiv_re), re.I)
def body_encoding(self):
return self._body_declared_encoding() or self._body_inferred_encoding()
return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding()
@memoizemethod('cache')
def _body_declared_encoding(self):

41
scrapy/http/response/text.py
View File

@ -16,27 +16,22 @@ class TextResponse(Response):
_ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I)
__slots__ = ['_encoding']
__slots__ = ['_encoding', '_body_inferred_encoding']
def __init__(self, url, status=200, headers=None, body=None, meta=None, flags=None, encoding=None):
def __init__(self, url, status=200, headers=None, body=None, meta=None, \
flags=None, encoding=None):
self._encoding = encoding
if isinstance(body, unicode):
if encoding is None:
clsname = self.__class__.__name__
raise TypeError("To instantiate a %s with unicode body you must specify the encoding" % clsname)
body = body.encode(encoding)
Response.__init__(self, url, status, headers, body, meta, flags)
self._body_inferred_encoding = None
super(TextResponse, self).__init__(url, status, headers, body, meta, flags)
def set_body(self, body):
if isinstance(body, str):
self._body = body
elif isinstance(body, unicode):
def _set_body(self, body):
if isinstance(body, unicode):
if self._encoding is None:
raise TypeError("To instantiate a %s with unicode body you " \
"must specify the encoding" % self.__class__.__name__)
self._body = body.encode(self._encoding)
elif body is None:
self._body = None
else:
raise TypeError("Request body must either str, unicode or None. Got: '%s'" % type(body).__name__)
body = property(lambda x: x._body, set_body)
super(TextResponse, self)._set_body(body)
def replace(self, *args, **kwargs):
kwargs.setdefault('encoding', getattr(self, '_encoding', None))
@ -59,19 +54,17 @@ class TextResponse(Response):
@memoizemethod('cache')
def body_as_unicode(self):
"""Return body as unicode"""
possible_encodings = (self._encoding, self.headers_encoding(), self._body_declared_encoding())
possible_encodings = (self._encoding, self.headers_encoding(), \
self._body_declared_encoding())
dammit = UnicodeDammit(self.body, possible_encodings)
self.cache['body_inferred_encoding'] = dammit.originalEncoding
# XXX: sometimes dammit.unicode fails, even when it recognizes the encoding correctly
self._body_inferred_encoding = dammit.originalEncoding
return dammit.unicode
def body_encoding(self):
return self._body_inferred_encoding()
def _body_inferred_encoding(self):
if 'body_inferred_encoding' not in self.cache:
if self._body_inferred_encoding is None:
self.body_as_unicode()
return self.cache['body_inferred_encoding']
return self._body_inferred_encoding
def _body_declared_encoding(self):
# implemented in subclasses (XmlResponse, HtmlResponse)
return None

2
scrapy/http/response/xml.py
View File

@ -19,7 +19,7 @@ class XmlResponse(TextResponse):
XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I)
def body_encoding(self):
return self._body_declared_encoding() or self._body_inferred_encoding()
return self._body_declared_encoding() or super(XmlResponse, self).body_encoding()
@memoizemethod('cache')
def _body_declared_encoding(self):