diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ea64d1599..2e92961a9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -445,10 +445,10 @@ Response objects .. attribute:: Response.body - A str containing the body of this Response. Keep in mind that Response.body - is always a str. If you want the unicode version use - :meth:`TextResponse.body_as_unicode` (only available in - :class:`TextResponse` and subclasses). + The body of this Response. Keep in mind that Response.body + is always a bytes object. If you want the unicode version use + :attr:`TextResponse.txt` (only available in :class:`TextResponse` + and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -542,6 +542,21 @@ TextResponse objects :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: + .. attribute:: TextResponse.text + + Response body, as unicode. + + The same as ``response.body.decode(response.encoding)``, but the + result is cached after the first call, so you can access + ``response.text`` multiple times without extra overhead. + + .. note:: + + ``unicode(response.body)`` is not a correct way to convert response + body to unicode: you would be using the system default encoding + (typically `ascii`) instead of the response encoding. + + .. attribute:: TextResponse.encoding A string with the encoding of this response. The encoding is resolved by @@ -568,20 +583,6 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.body_as_unicode() - - Returns the body of the response as unicode. This is equivalent to:: - - response.body.decode(response.encoding) - - But **not** equivalent to:: - - unicode(response.body) - - Since, in the latter case, you would be using the system default encoding - (typically `ascii`) to convert the body to unicode, instead of the response - encoding. - .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: @@ -594,6 +595,11 @@ TextResponse objects response.css('p') + .. method:: TextResponse.body_as_unicode() + + The same as :attr:`text`, but available as a method. This method is + kept for backwards compatibility; please prefer ``response.text``. + HtmlResponse objects -------------------- diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 6b543b823..da373eca2 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object): Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - body = response.body_as_unicode()[:self.lookup_bytes] + body = response.text[:self.lookup_bytes] return _has_ajaxcrawlable_meta(body) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..d4a33dc36 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object): def _parse_robots(self, response, netloc): rp = robotparser.RobotFileParser(response.url) body = '' - if hasattr(response, 'body_as_unicode'): - body = response.body_as_unicode() + if hasattr(response, 'text'): + body = response.text else: # last effort try try: body = response.body.decode('utf-8') diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 5501634d3..2862dc096 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -64,8 +64,8 @@ def _urlencode(seq, enc): def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ - text = response.body_as_unicode() - root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response)) + root = create_root_node(response.text, lxml.html.HTMLParser, + base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1c416bf82..9c667ab7e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -67,6 +67,11 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody + @property + def text(self): + """ Body as unicode """ + return self.body_as_unicode() + def urljoin(self, url): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 5d77f7624..15f3d26df 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref): response = _response_from_text(text, st) if response is not None: - text = response.body_as_unicode() + text = response.text kwargs.setdefault('base_url', response.url) self.response = response diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b0688791e..73857b410 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): if not unicode: return obj.body elif isinstance(obj, TextResponse): - return obj.body_as_unicode() + return obj.text else: return obj.body.decode('utf-8') elif isinstance(obj, six.text_type): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c4ad52f14..73db2641e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary() def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] text = _noscript_re.sub(u'', text) text = _script_re.sub(u'', text) _metaref_cache[response] = html.get_meta_refresh(text, response.url, diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f2c02bff..baf6ef1bf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -55,12 +55,11 @@ class TestSpider(Spider): def parse_item(self, response): item = self.item_cls() - body = response.body_as_unicode() - m = self.name_re.search(body) + m = self.name_re.search(response.text) if m: item['name'] = m.group(1) item['url'] = response.url - m = self.price_re.search(body) + m = self.price_re.search(response.text) if m: item['price'] = m.group(1) return item diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 710a5b29d..c7f36687a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) + assert isinstance(response.text, six.text_type) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) + self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): self.assertEqual(response.encoding, resolve_encoding(encoding)) @@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest): self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) self.assertEqual(r1.body_as_unicode(), unicode_string) + # check response.text + self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertEqual(r1.text, unicode_string) + def test_encoding(self): r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") @@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd') + self.assertEqual(r6.text, u'WORD\ufffd\ufffd') def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.body_as_unicode() in indistint order doesn't affect final + # response.text in indistint order doesn't affect final # values for encoding and decoded body. url = 'http://example.com' body = b"\xef\xbb\xbfWORD" @@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): @@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest): r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'\ufffd' in r.text, repr(r.text) + assert u'PREFIX' in r.text, repr(r.text) + assert u'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', \ body=b'\xf0value') - assert u'value' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') - #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page"