From 877057fac0d47e5ece95a55594706c91c8855883 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:00:09 +0500 Subject: [PATCH 1/7] initial response.follow implementation --- docs/intro/overview.rst | 3 +- docs/intro/tutorial.rst | 46 +++++++++++++++++++------ docs/topics/request-response.rst | 4 +++ scrapy/http/response/text.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7195017ff..1da1a4059 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination:: next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield response.follow(next_page, self.parse) Put this in a text file, name it to something like ``quotes_spider.py`` diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3dc5ad2ed..d47bf69e5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. +As a shortcut for creating Request objects you can use +:meth:`response.follow ` method:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +Unlike scrapy.Request, ``response.follow`` supports +relative URLs directly; you can also pass a selector to it instead of +a string. Note that ``response.follow`` just returns a Request instance; +you still have to yield this Request. + More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: - import scrapy @@ -568,15 +595,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author + a::attr(href)').extract(): - yield scrapy.Request(response.urljoin(href), - callback=self.parse_author) + for href in response.css('.author + a::attr(href)'): + yield response.follow(href, self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr(href)').extract_first() - if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. +Here we're passing callbacks to ``response.follow`` as positional arguments +to make the code shorter; it also works for ``scrapy.Request``. + The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument:: next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, self.parse) + yield response.follow(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1fdd26043..71050fddd 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -683,6 +683,10 @@ TextResponse objects response.css('p') + .. method:: TextResponse.follow(url, ...) + + Return a scrapy.Request instance to follow a link ``url``. + .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5a6507aa8..1718b1f3b 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst import six from six.moves.urllib.parse import urljoin +import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding + +from scrapy.link import Link +from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str @@ -116,3 +120,58 @@ class TextResponse(Response): def css(self, query): return self.selector.css(query) + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding=None, priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object (e.g. a link extractor result); + * attribute Selector (not SelectorList) - e.g. + ``response.css('a::attr(href)')[0]`` or + ``response.xpath('//img/@src')[0]``. + * a Selector for ```` element, e.g. + ``response.css('a.my_link')[0]``. + """ + if isinstance(url, Link): + url = url.url + elif isinstance(url, parsel.Selector): + url = _url_from_selector(url) + elif isinstance(url, parsel.SelectorList): + raise ValueError("Please pass either string") + + + encoding = self.encoding if encoding is None else encoding + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) + + +def _url_from_selector(sel): + # type: (parsel.Selector) -> str + if isinstance(sel.root, six.string_types): + # e.g. ::attr(href) result + return sel.root + if not hasattr(sel.root, 'tag'): + raise ValueError("Unsupported selector: %s" % sel) + if sel.root.tag != 'a': + raise ValueError("Only elements are supported; got <%s>" % + sel.root.tag) + href = sel.root.get('href') + if href is None: + raise ValueError(" element has no href attribute: %s" % sel) + return href From 71dd5d0bf9d0c41d70e72b0fd0a89528ef246065 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:11:08 +0500 Subject: [PATCH 2/7] strip URL extracted from selectors (as per html5 standard) --- scrapy/http/response/text.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1718b1f3b..5bfd2debb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,10 +142,9 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url) + url = _url_from_selector(url).strip() elif isinstance(url, parsel.SelectorList): - raise ValueError("Please pass either string") - + raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding url = self.urljoin(url) From 608c3f0c452bd508aa24bc9d4bf759e0b11e1683 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:17:41 +0500 Subject: [PATCH 3/7] handle whitespace in response.follow; add tests --- scrapy/http/response/text.py | 7 ++- tests/__init__.py | 7 ++- tests/test_http_response.py | 105 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5bfd2debb..6eacfbd35 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding +from w3lib.html import strip_html5_whitespace from scrapy.link import Link from scrapy.http.request import Request @@ -142,7 +143,7 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url).strip() + url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") @@ -164,7 +165,7 @@ def _url_from_selector(sel): # type: (parsel.Selector) -> str if isinstance(sel.root, six.string_types): # e.g. ::attr(href) result - return sel.root + return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): raise ValueError("Unsupported selector: %s" % sel) if sel.root.tag != 'a': @@ -173,4 +174,4 @@ def _url_from_selector(sel): href = sel.root.get('href') if href is None: raise ValueError(" element has no href attribute: %s" % sel) - return href + return strip_html5_whitespace(href) diff --git a/tests/__init__.py b/tests/__init__.py index d940f28ea..c2e4fd2bf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,9 +26,12 @@ try: except ImportError: import mock -tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'sample_data') + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - return open(path, 'rb').read() + with open(path, 'rb') as f: + return f.read() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 9df3bf6e7..2a9baf5ed 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import unittest import six @@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse, from scrapy.selector import Selector from scrapy.utils.python import to_native_str from scrapy.exceptions import NotSupported +from scrapy.link import Link +from tests import get_testdata class BaseResponseTest(unittest.TestCase): @@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + def test_html_encoding(self): body = b"""Some page @@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) + def assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def test_follow_url_absolute(self): + self.assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self.assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self.assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self.assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_url(self): + self.assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self.assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self.assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self.assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self.assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self.assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class XmlResponseTest(TextResponseTest): From 2674f317df9e4970f5953db3a6df04331246d8c9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:39:47 +0500 Subject: [PATCH 4/7] Response.follow --- scrapy/http/response/__init__.py | 29 +++++ scrapy/http/response/text.py | 27 ++-- tests/test_http_response.py | 204 +++++++++++++++---------------- 3 files changed, 143 insertions(+), 117 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 58ad414f1..e5fb4eef8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst """ from six.moves.urllib.parse import urljoin +from scrapy.http.request import Request from scrapy.http.headers import Headers +from scrapy.link import Link from scrapy.utils.trackref import object_ref from scrapy.http.common import obsolete_setter from scrapy.exceptions import NotSupported @@ -101,3 +103,30 @@ class Response(object_ref): is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding='utf-8', priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object. + """ + if isinstance(url, Link): + url = url.url + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6eacfbd35..3c360bcf9 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,25 +140,22 @@ class TextResponse(Response): * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. """ - if isinstance(url, Link): - url = url.url - elif isinstance(url, parsel.Selector): + if isinstance(url, parsel.Selector): url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") - encoding = self.encoding if encoding is None else encoding - url = self.urljoin(url) - return Request(url, callback, - method=method, - headers=headers, - body=body, - cookies=cookies, - meta=meta, - encoding=encoding, - priority=priority, - dont_filter=dont_filter, - errback=errback) + return super(TextResponse, self).follow(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback + ) def _url_from_selector(sel): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2a9baf5ed..e64d0eeba 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -143,6 +143,38 @@ class BaseResponseTest(unittest.TestCase): r.css('body') r.xpath('//body') + def test_follow_url_absolute(self): + self._assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self._assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self._assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_whitespace_url(self): + self._assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self._assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def _assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + class TextResponseTest(BaseResponseTest): @@ -354,16 +386,81 @@ class TextResponseTest(BaseResponseTest): absolute = 'http://www.example.com/elsewhere/test' self.assertEqual(joined, absolute) + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self._assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self._assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self._assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self._assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse - def _links_response(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - resp = self.response_class('http://example.com/index', body=body) - return resp - def test_html_encoding(self): body = b"""Some page @@ -396,103 +493,6 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) - def assert_followed_url(self, follow_obj, target_url, response=None): - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) - return req - - def test_follow_url_absolute(self): - self.assert_followed_url('http://foo.example.com', - 'http://foo.example.com') - - def test_follow_url_relative(self): - self.assert_followed_url('foo', - 'http://example.com/foo') - - def test_follow_link(self): - self.assert_followed_url(Link('http://example.com/foo'), - 'http://example.com/foo') - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - 'http://example.com/sample2.html', - 'http://example.com/sample3.html', - 'http://example.com/sample3.html', - 'http://www.google.com/something', - 'http://example.com/innertag.html' - ] - - # select elements - for sellist in [resp.css('a'), resp.xpath('//a')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # href attributes should work - for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css('img::attr(src)'): - self.assert_followed_url(src, 'http://example.com/sample2.jpg') - - def test_follow_whitespace_url(self): - self.assert_followed_url('foo ', - 'http://example.com/foo%20') - - def test_follow_whitespace_link(self): - self.assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') - - def test_follow_whitespace_selector(self): - resp = self.response_class( - 'http://example.com', - body=b'''click me''' - ) - self.assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self.assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) - - def test_follow_encoding(self): - resp1 = self.response_class( - 'http://example.com', - encoding='utf8', - body='click me'.encode('utf8') - ) - req = self.assert_followed_url( - resp1.css('a')[0], - 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', - response=resp1, - ) - self.assertEqual(req.encoding, 'utf8') - - resp2 = self.response_class( - 'http://example.com', - encoding='cp1251', - body='click me'.encode('cp1251') - ) - req = self.assert_followed_url( - resp2.css('a')[0], - 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', - response=resp2, - ) - self.assertEqual(req.encoding, 'cp1251') - class XmlResponseTest(TextResponseTest): From 160da6abab8954906181ce69593ff6e84d950ac1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:41:53 +0500 Subject: [PATCH 5/7] fixed tests in Python 2 --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e64d0eeba..fa74b468b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -411,8 +411,8 @@ class TextResponseTest(BaseResponseTest): def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegexp(ValueError, 'SelectorList', + resp.follow, resp.css('a')) def test_follow_selector_attribute(self): resp = self._links_response() @@ -435,7 +435,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body='click me'.encode('utf8') + body=u'click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -447,7 +447,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body='click me'.encode('cp1251') + body=u'click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], From 5b79c6a679b66868c89302a1693e5dedc62b6f61 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 00:06:52 +0500 Subject: [PATCH 6/7] DOC document response.follow methods; expand the tutorial --- docs/intro/tutorial.rst | 41 +++++++++++++++++++++++++------- docs/topics/request-response.rst | 7 +++--- scrapy/http/response/__init__.py | 15 ++++++------ scrapy/http/response/text.py | 18 +++++++------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index d47bf69e5..3b3bd8d21 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary:: >>> Extracting data in our spider ------------------------------- +----------------------------- Let's get back to our spider. Until now, it doesn't extract any data in particular, just saves the whole HTML page to a local file. Let's integrate the @@ -551,8 +551,14 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. + +.. _response-follow-example: + +A shortcut for creating Requests +-------------------------------- + As a shortcut for creating Request objects you can use -:meth:`response.follow ` method:: +:meth:`response.follow `:: import scrapy @@ -571,13 +577,32 @@ As a shortcut for creating Request objects you can use 'tags': quote.css('div.tags a.tag::text').extract(), } - for href in response.css('li.next a::attr(href)'): - yield response.follow(href, callback=self.parse) + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) -Unlike scrapy.Request, ``response.follow`` supports -relative URLs directly; you can also pass a selector to it instead of -a string. Note that ``response.follow`` just returns a Request instance; -you still have to yield this Request. +Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no +need to call urljoin. Note that ``response.follow`` just returns a Request +instance; you still have to yield this Request. + +You can also pass a selector to ``response.follow`` instead of a string; +this selector should extract necessary attributes:: + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +For ```` elements there is a shortcut: ``response.follow`` uses their href +attribute automatically. So the code can be shortened further:: + + for a in response.css('li.next a'): + yield response.follow(a, callback=self.parse) + +.. note:: + + ``response.follow(response.css('li.next a'))`` is not valid because + ``response.css`` returns a list-like object with selectors for all results, + not a single selector. A ``for`` loop like in the example above, or + ``response.follow(response.css('li.next a')[0])`` is fine. More examples and patterns -------------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 71050fddd..3e80f18b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -597,6 +597,9 @@ Response objects urlparse.urljoin(response.url, url) + .. automethod:: Response.follow + + .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin .. _topics-request-response-ref-response-subclasses: @@ -683,9 +686,7 @@ TextResponse objects response.css('p') - .. method:: TextResponse.follow(url, ...) - - Return a scrapy.Request instance to follow a link ``url``. + .. automethod:: TextResponse.follow .. method:: TextResponse.body_as_unicode() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index e5fb4eef8..434d87eab 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -109,13 +109,14 @@ class Response(object_ref): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + not only an absolute URL. + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` + method which supports selectors in addition to absolute/relative URLs + and Link objects. """ if isinstance(url, Link): url = url.url diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 3c360bcf9..6415e191a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace -from scrapy.link import Link from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url @@ -127,18 +126,19 @@ class TextResponse(Response): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object (e.g. a link extractor result); - * attribute Selector (not SelectorList) - e.g. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be not only an absolute URL, but also + + * a relative URL; + * a scrapy.link.Link object (e.g. a link extractor result); + * an attribute Selector (not SelectorList) - e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]``. * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. + + See :ref:`response-follow-example` for usage examples. """ if isinstance(url, parsel.Selector): url = _url_from_selector(url) From fade5763af3d03f076f3317589038201bbdeccaf Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 02:02:50 +0500 Subject: [PATCH 7/7] TST more response.follow tests --- tests/test_http_response.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index fa74b468b..924bb7979 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -414,11 +414,24 @@ class TextResponseTest(BaseResponseTest): self.assertRaisesRegexp(ValueError, 'SelectorList', resp.follow, resp.css('a')) + def test_follow_selector_invalid(self): + resp = self._links_response() + self.assertRaisesRegexp(ValueError, 'Unsupported', + resp.follow, resp.xpath('count(//div)')[0]) + def test_follow_selector_attribute(self): resp = self._links_response() for src in resp.css('img::attr(src)'): self._assert_followed_url(src, 'http://example.com/sample2.jpg') + def test_follow_selector_no_href(self): + resp = self.response_class( + url='http://example.com', + body=b'click me', + ) + self.assertRaisesRegexp(ValueError, 'no href', + resp.follow, resp.css('a')[0]) + def test_follow_whitespace_selector(self): resp = self.response_class( 'http://example.com',