RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

response.text. Fixes GH-1729.

This commit is contained in:
Mikhail Korobov 2016-01-27 01:28:11 +05:00
parent 85f0596c43
commit 4bcbb77bcc
10 changed files with 58 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
docs/topics/request-response.rst
View File

@ -445,10 +445,10 @@ Response objects
.. attribute:: Response.body
A str containing the body of this Response. Keep in mind that Response.body
is always a str. If you want the unicode version use
:meth:`TextResponse.body_as_unicode` (only available in
:class:`TextResponse` and subclasses).
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the unicode version use
:attr:`TextResponse.txt` (only available in :class:`TextResponse`
and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@ -542,6 +542,21 @@ TextResponse objects
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:
.. attribute:: TextResponse.text
Response body, as unicode.
The same as ``response.body.decode(response.encoding)``, but the
result is cached after the first call, so you can access
``response.text`` multiple times without extra overhead.
.. note::
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically `ascii`) instead of the response encoding.
.. attribute:: TextResponse.encoding
A string with the encoding of this response. The encoding is resolved by
@ -568,20 +583,6 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
.. method:: TextResponse.body_as_unicode()
Returns the body of the response as unicode. This is equivalent to::
response.body.decode(response.encoding)
But **not** equivalent to::
unicode(response.body)
Since, in the latter case, you would be using the system default encoding
(typically `ascii`) to convert the body to unicode, instead of the response
encoding.
.. method:: TextResponse.xpath(query)
A shortcut to ``TextResponse.selector.xpath(query)``::
@ -594,6 +595,11 @@ TextResponse objects
response.css('p')
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
kept for backwards compatibility; please prefer ``response.text``.
HtmlResponse objects
--------------------

2
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object):
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
body = response.body_as_unicode()[:self.lookup_bytes]
body = response.text[:self.lookup_bytes]
return _has_ajaxcrawlable_meta(body)

4
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object):
def _parse_robots(self, response, netloc):
rp = robotparser.RobotFileParser(response.url)
body = ''
if hasattr(response, 'body_as_unicode'):
body = response.body_as_unicode()
if hasattr(response, 'text'):
body = response.text
else: # last effort try
try:
body = response.body.decode('utf-8')

4
scrapy/http/request/form.py
View File

@ -64,8 +64,8 @@ def _urlencode(seq, enc):
def _get_form(response, formname, formid, formnumber, formxpath):
"""Find the form element """
text = response.body_as_unicode()
root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
root = create_root_node(response.text, lxml.html.HTMLParser,
base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)

5
scrapy/http/response/text.py
View File

@ -67,6 +67,11 @@ class TextResponse(Response):
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
@property
def text(self):
""" Body as unicode """
return self.body_as_unicode()
def urljoin(self, url):
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""

2
scrapy/selector/unified.py
View File

@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref):
response = _response_from_text(text, st)
if response is not None:
text = response.body_as_unicode()
text = response.text
kwargs.setdefault('base_url', response.url)
self.response = response

2
scrapy/utils/iterators.py
View File

@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
if not unicode:
return obj.body
elif isinstance(obj, TextResponse):
return obj.body_as_unicode()
return obj.text
else:
return obj.body.decode('utf-8')
elif isinstance(obj, six.text_type):

4
scrapy/utils/response.py
View File

@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary()
def get_base_url(response):
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
text = response.body_as_unicode()[0:4096]
text = response.text[0:4096]
_baseurl_cache[response] = html.get_base_url(text, response.url,
response.encoding)
return _baseurl_cache[response]
@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.body_as_unicode()[0:4096]
text = response.text[0:4096]
text = _noscript_re.sub(u'', text)
text = _script_re.sub(u'', text)
_metaref_cache[response] = html.get_meta_refresh(text, response.url,

5
tests/test_engine.py
View File

@ -55,12 +55,11 @@ class TestSpider(Spider):
def parse_item(self, response):
item = self.item_cls()
body = response.body_as_unicode()
m = self.name_re.search(body)
m = self.name_re.search(response.text)
if m:
item['name'] = m.group(1)
item['url'] = response.url
m = self.price_re.search(body)
m = self.price_re.search(response.text)
if m:
item['price'] = m.group(1)
return item

30
tests/test_http_response.py
View File

@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase):
body_bytes = body
assert isinstance(response.body, bytes)
assert isinstance(response.text, six.text_type)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
self.assertEqual(response.body_as_unicode(), body_unicode)
self.assertEqual(response.text, body_unicode)
def _assert_response_encoding(self, response, encoding):
self.assertEqual(response.encoding, resolve_encoding(encoding))
@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest):
self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
self.assertEqual(r1.body_as_unicode(), unicode_string)
# check response.text
self.assertTrue(isinstance(r1.text, six.text_type))
self.assertEqual(r1.text, unicode_string)
def test_encoding(self):
r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest):
headers={"Content-type": ["text/html; charset=utf-8"]},
body=b"\xef\xbb\xbfWORD\xe3\xab")
self.assertEqual(r6.encoding, 'utf-8')
self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')
self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.body_as_unicode() in indistint order doesn't affect final
# response.text in indistint order doesn't affect final
# values for encoding and decoded body.
url = 'http://example.com'
body = b"\xef\xbb\xbfWORD"
@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest):
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
self.assertEqual(response.encoding, 'utf-8')
self.assertEqual(response.body_as_unicode(), u'WORD')
self.assertEqual(response.text, u'WORD')
response = self.response_class(url, body=body)
self.assertEqual(response.body_as_unicode(), u'WORD')
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.encoding, 'utf-8')
# Body caching sideeffect isn't triggered when encoding is declared in
@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest):
# body
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.encoding, 'utf-8')
self.assertEqual(response.body_as_unicode(), u'WORD')
self.assertEqual(response.text, u'WORD')
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.body_as_unicode(), u'WORD')
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.encoding, 'utf-8')
def test_replace_wrong_encoding(self):
@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest):
r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
# XXX: Policy for replacing invalid chars may suffer minor variations
# but it should always contain the unicode replacement char (u'\ufffd')
assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
assert u'\ufffd' in r.text, repr(r.text)
assert u'PREFIX' in r.text, repr(r.text)
assert u'SUFFIX' in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class("http://example.com", encoding='utf-8', \
body=b'\xf0<span>value</span>')
assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())
assert u'<span>value</span>' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
#r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX')
#assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
#r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
#assert u'\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"