From 5f168cd459cfc026de1e4d0b43c45ea740fe1dd5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 2 Oct 2019 14:08:08 -0300 Subject: [PATCH 01/18] Response.follow_all --- docs/topics/request-response.rst | 4 ++ scrapy/http/response/__init__.py | 63 +++++++++++++++++----- scrapy/http/response/text.py | 61 ++++++++++++++++++--- tests/test_http_response.py | 91 ++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 20 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..9fe3c7518 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -701,6 +701,8 @@ Response objects .. automethod:: Response.follow + .. automethod:: Response.follow_all + .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin @@ -790,6 +792,8 @@ TextResponse objects .. automethod:: TextResponse.follow + .. automethod:: TextResponse.follow_all + .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index b0a526b72..96359705e 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -113,8 +113,8 @@ class Response(object_ref): It accepts the same arguments as ``Request.__init__`` method, but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, not only an absolute URL. - - :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` method which supports selectors in addition to absolute/relative URLs and Link objects. """ @@ -123,14 +123,51 @@ class Response(object_ref): elif url is None: raise ValueError("url can't be None") url = self.urljoin(url) - return Request(url, callback, - method=method, - headers=headers, - body=body, - cookies=cookies, - meta=meta, - encoding=encoding, - priority=priority, - dont_filter=dont_filter, - errback=errback, - cb_kwargs=cb_kwargs) + return Request( + url=url, + callback=callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback, + cb_kwargs=cb_kwargs, + ) + + def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding='utf-8', priority=0, + dont_filter=False, errback=None, cb_kwargs=None): + # type: (...) -> Generator[Request, None, None] + """ + Return an iterable of :class:`~.Request` instance to follow all links + in ``urls``. It accepts the same arguments as ``Request.__init__`` method, + but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects + not only absolute URLs. + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all` + method which supports selectors in addition to absolute/relative URLs + and Link objects. + """ + if not hasattr(urls, '__iter__'): + raise TypeError("'urls' argument must be an iterable") + return ( + self.follow( + url=url, + callback=callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback, + cb_kwargs=cb_kwargs, + ) + for url in urls + ) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 339913d4e..81e3f9b28 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace -from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str @@ -44,7 +43,7 @@ class TextResponse(Response): if isinstance(body, six.text_type): if self._encoding is None: raise TypeError('Cannot convert unicode body - %s has no encoding' % - type(self).__name__) + type(self).__name__) self._body = body.encode(self._encoding) else: super(TextResponse, self)._set_body(body) @@ -90,8 +89,8 @@ class TextResponse(Response): if self._cached_benc is None: content_type = to_native_str(self.headers.get(b'Content-Type', b'')) benc, ubody = html_to_unicode(content_type, self.body, - auto_detect_fun=self._auto_detect_fun, - default_encoding=self._DEFAULT_ENCODING) + auto_detect_fun=self._auto_detect_fun, + default_encoding=self._DEFAULT_ENCODING) self._cached_benc = benc self._cached_ubody = ubody return self._cached_benc @@ -129,7 +128,7 @@ class TextResponse(Response): Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, but ``url`` can be not only an absolute URL, but also - + * a relative URL; * a scrapy.link.Link object (e.g. a link extractor result); * an attribute Selector (not SelectorList) - e.g. @@ -137,7 +136,7 @@ class TextResponse(Response): ``response.xpath('//img/@src')[0]``. * a Selector for ```` or ```` element, e.g. ``response.css('a.my_link')[0]``. - + See :ref:`response-follow-example` for usage examples. """ if isinstance(url, parsel.Selector): @@ -145,7 +144,9 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow(url, callback, + return super(TextResponse, self).follow( + url=url, + callback=callback, method=method, headers=headers, body=body, @@ -158,6 +159,52 @@ class TextResponse(Response): cb_kwargs=cb_kwargs, ) + def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding=None, priority=0, + dont_filter=False, errback=None, cb_kwargs=None, + css=None, xpath=None): + # type: (...) -> Generator[Request, None, None] + """ + A generator that produces :class:`~.Request` instances to follow all + links in ``urls``. It accepts the same arguments as the :class:`~.Request` + initializer, except that each ``urls`` element does not need to be an absolute + URL, it can be any of the following: + + * a relative URL; + * a scrapy.link.Link object (e.g. a link extractor result); + * an attribute Selector (not SelectorList) - e.g. + ``response.css('a::attr(href)')[0]`` or + ``response.xpath('//img/@src')[0]``. + * a Selector for ```` or ```` element, e.g. + ``response.css('a.my_link')[0]``. + + In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction + within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted). + """ + if len(list(filter(None, (urls, css, xpath)))) > 1: + raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}') + if css: + urls = self.css(css) + elif xpath: + urls = self.xpath(xpath) + return ( + self.follow( + url=url, + callback=callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback, + cb_kwargs=cb_kwargs, + ) + for url in urls + ) + def _url_from_selector(sel): # type: (parsel.Selector) -> str diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cd5c3486e..134856c78 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -143,6 +143,8 @@ class BaseResponseTest(unittest.TestCase): r.css('body') r.xpath('//body') + # Response.follow + def test_follow_url_absolute(self): self._assert_followed_url('http://foo.example.com', 'http://foo.example.com') @@ -166,6 +168,64 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') + + # Response.follow_all + + def test_follow_all_absolute(self): + url_list = ['http://example.org', 'http://www.example.org', + 'http://example.com', 'http://www.example.com'] + self._assert_followed_all_urls(url_list, url_list) + + def test_follow_all_relative(self): + relative = ['foo', 'bar', 'foo/bar', 'bar/foo'] + absolute = [ + 'http://example.com/foo', + 'http://example.com/bar', + 'http://example.com/foo/bar', + 'http://example.com/bar/foo', + ] + self._assert_followed_all_urls(relative, absolute) + + def test_follow_all_links(self): + absolute = [ + 'http://example.com/foo', + 'http://example.com/bar', + 'http://example.com/foo/bar', + 'http://example.com/bar/foo', + ] + links = map(Link, absolute) + self._assert_followed_all_urls(links, absolute) + + def test_follow_all_invalid(self): + r = self.response_class("http://example.com") + with self.assertRaises(TypeError): + list(r.follow_all(None)) + with self.assertRaises(TypeError): + list(r.follow_all(12345)) + with self.assertRaises(ValueError): + list(r.follow_all([None])) + + def test_follow_all_whitespace(self): + relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo '] + absolute = [ + 'http://example.com/foo%20', + 'http://example.com/bar%20', + 'http://example.com/foo/bar%20', + 'http://example.com/bar/foo%20', + ] + self._assert_followed_all_urls(relative, absolute) + + def test_follow_all_whitespace_links(self): + absolute = [ + 'http://example.com/foo ', + 'http://example.com/bar ', + 'http://example.com/foo/bar ', + 'http://example.com/bar/foo ', + ] + links = map(Link, absolute) + expected = [u.replace(' ', '%20') for u in absolute] + self._assert_followed_all_urls(links, expected) + def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() @@ -173,6 +233,14 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(req.url, target_url) return req + def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): + if response is None: + response = self._links_response() + followed = response.follow_all(follow_obj) + for req, target in zip(followed, target_urls): + self.assertEqual(req.url, target) + yield req + def _links_response(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') resp = self.response_class('http://example.com/index', body=body) @@ -483,6 +551,29 @@ class TextResponseTest(BaseResponseTest): ) self.assertEqual(req.encoding, 'cp1251') + def test_follow_all_css(self): + expected = [ + 'http://example.com/sample3.html', + 'http://example.com/innertag.html', + ] + response = self._links_response() + extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] + self.assertEqual(expected, extracted) + + def test_follow_all_xpath(self): + expected = [ + 'http://example.com/sample3.html', + 'http://example.com/innertag.html', + ] + response = self._links_response() + extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') + self.assertEqual(expected, [r.url for r in extracted]) + + def test_follow_all_exception(self): + response = self._links_response() + with self.assertRaises(ValueError): + response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') + class HtmlResponseTest(TextResponseTest): From e1fa1fd8ad62cb8958d21d3a33648ed4de6af757 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 10 Oct 2019 00:36:38 -0300 Subject: [PATCH 02/18] TextResponse.follow_all: skip invalid links --- scrapy/http/response/text.py | 26 +++++++--- .../sgml_linkextractor_no_href.html | 25 ++++++++++ tests/test_http_response.py | 47 ++++++++++++++++--- 3 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 tests/sample_data/link_extractor/sgml_linkextractor_no_href.html diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 81e3f9b28..ccacce550 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -180,13 +180,27 @@ class TextResponse(Response): In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted). + + Note that when using the ``css`` or ``xpath`` parameters, this method will not produce + requests for selectors from which links cannot be obtained (for instance, anchor tags + without ``href`` attribute) """ - if len(list(filter(None, (urls, css, xpath)))) > 1: - raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}') - if css: - urls = self.css(css) - elif xpath: - urls = self.xpath(xpath) + arg_count = len(list(filter(None, (urls, css, xpath)))) + if arg_count != 1: + raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}') + if not urls: + urls = [] + if css: + selector_method = getattr(self, 'css') + expression = css + elif xpath: + selector_method = getattr(self, 'xpath') + expression = xpath + for selector in selector_method(expression): + try: + urls.append(_url_from_selector(selector)) + except ValueError: + pass return ( self.follow( url=url, diff --git a/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html new file mode 100644 index 000000000..0b01cede8 --- /dev/null +++ b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html @@ -0,0 +1,25 @@ + + + + Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method + + + +
+ “The world as we have created it is a process of our + thinking. It cannot be changed without changing our thinking.” + + by Albert Einstein + (about) + + +
+ + + \ No newline at end of file diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 134856c78..6d3c5cb9d 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -198,12 +198,20 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_invalid(self): r = self.response_class("http://example.com") - with self.assertRaises(TypeError): - list(r.follow_all(None)) - with self.assertRaises(TypeError): - list(r.follow_all(12345)) - with self.assertRaises(ValueError): - list(r.follow_all([None])) + if self.response_class == Response: + with self.assertRaises(TypeError): + list(r.follow_all(urls=None)) + with self.assertRaises(TypeError): + list(r.follow_all(urls=12345)) + with self.assertRaises(ValueError): + list(r.follow_all(urls=[None])) + else: + with self.assertRaises(ValueError): + list(r.follow_all(urls=None)) + with self.assertRaises(TypeError): + list(r.follow_all(urls=12345)) + with self.assertRaises(ValueError): + list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo '] @@ -246,6 +254,11 @@ class BaseResponseTest(unittest.TestCase): resp = self.response_class('http://example.com/index', body=body) return resp + def _links_response_no_href(self): + body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + class TextResponseTest(BaseResponseTest): @@ -560,6 +573,16 @@ class TextResponseTest(BaseResponseTest): extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] self.assertEqual(expected, extracted) + def test_follow_all_css_skip_invalid(self): + expected = [ + 'http://example.com/page/1/', + 'http://example.com/page/3/', + 'http://example.com/page/4/', + ] + response = self._links_response_no_href() + extracted = [r.url for r in response.follow_all(css='.pagination a')] + self.assertEqual(expected, extracted) + def test_follow_all_xpath(self): expected = [ 'http://example.com/sample3.html', @@ -569,7 +592,17 @@ class TextResponseTest(BaseResponseTest): extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') self.assertEqual(expected, [r.url for r in extracted]) - def test_follow_all_exception(self): + def test_follow_all_xpath_skip_invalid(self): + expected = [ + 'http://example.com/page/1/', + 'http://example.com/page/3/', + 'http://example.com/page/4/', + ] + response = self._links_response_no_href() + extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')] + self.assertEqual(expected, extracted) + + def test_follow_all_too_many_arguments(self): response = self._links_response() with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') From b970851299bf561af7ff0bb21caca9b6c3f296af Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 14 Oct 2019 13:35:06 -0300 Subject: [PATCH 03/18] Update scrapy/http/response/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- scrapy/http/response/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 96359705e..cdaababac 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -143,7 +143,7 @@ class Response(object_ref): dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Generator[Request, None, None] """ - Return an iterable of :class:`~.Request` instance to follow all links + Return an iterable of :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as ``Request.__init__`` method, but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects not only absolute URLs. From ba840c5a6bea95333b3b78fb767ac6f092d02177 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 14 Oct 2019 13:35:24 -0300 Subject: [PATCH 04/18] Update scrapy/http/response/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- scrapy/http/response/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index cdaababac..92fa01621 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -145,7 +145,7 @@ class Response(object_ref): """ Return an iterable of :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as ``Request.__init__`` method, - but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects + but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, not only absolute URLs. :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all` From 498d33aac37d5d7a0955e9b409fc3d5ff8627dc0 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 14 Oct 2019 13:35:54 -0300 Subject: [PATCH 05/18] Update scrapy/http/response/text.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- scrapy/http/response/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index ccacce550..bb5a4eb9d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -183,7 +183,7 @@ class TextResponse(Response): Note that when using the ``css`` or ``xpath`` parameters, this method will not produce requests for selectors from which links cannot be obtained (for instance, anchor tags - without ``href`` attribute) + without an ``href`` attribute) """ arg_count = len(list(filter(None, (urls, css, xpath)))) if arg_count != 1: From c7c54f5453eaa16f2b6d6441be68b0a272606ec9 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 14 Oct 2019 13:47:44 -0300 Subject: [PATCH 06/18] Update scrapy/http/response/text.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- scrapy/http/response/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index bb5a4eb9d..ecb582b4a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -179,7 +179,7 @@ class TextResponse(Response): ``response.css('a.my_link')[0]``. In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction - within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted). + within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). Note that when using the ``css`` or ``xpath`` parameters, this method will not produce requests for selectors from which links cannot be obtained (for instance, anchor tags From 9d5398e7f2834940cbf9c2efa312bf5d96ffba98 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 14 Oct 2019 13:57:46 -0300 Subject: [PATCH 07/18] TextResponse.follow_all: improve docs --- scrapy/http/response/text.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index ecb582b4a..b2907baa4 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -129,13 +129,14 @@ class TextResponse(Response): It accepts the same arguments as ``Request.__init__`` method, but ``url`` can be not only an absolute URL, but also - * a relative URL; - * a scrapy.link.Link object (e.g. a link extractor result); - * an attribute Selector (not SelectorList) - e.g. + * a relative URL + * a :class:`~scrapy.link.Link` object, e.g. the result of + :ref:`topics-link-extractors` + * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + ``response.css('a.my_link')[0]`` + * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or - ``response.xpath('//img/@src')[0]``. - * a Selector for ```` or ```` element, e.g. - ``response.css('a.my_link')[0]``. + ``response.xpath('//img/@src')[0]`` See :ref:`response-follow-example` for usage examples. """ @@ -170,13 +171,14 @@ class TextResponse(Response): initializer, except that each ``urls`` element does not need to be an absolute URL, it can be any of the following: - * a relative URL; - * a scrapy.link.Link object (e.g. a link extractor result); - * an attribute Selector (not SelectorList) - e.g. + * a relative URL + * a :class:`~scrapy.link.Link` object, e.g. the result of + :ref:`topics-link-extractors` + * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + ``response.css('a.my_link')[0]`` + * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or - ``response.xpath('//img/@src')[0]``. - * a Selector for ```` or ```` element, e.g. - ``response.css('a.my_link')[0]``. + ``response.xpath('//img/@src')[0]`` In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). @@ -187,7 +189,7 @@ class TextResponse(Response): """ arg_count = len(list(filter(None, (urls, css, xpath)))) if arg_count != 1: - raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}') + raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath') if not urls: urls = [] if css: From 2a4d4a466aa6cede4829b62c7287332a627877ed Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Oct 2019 11:52:12 -0300 Subject: [PATCH 08/18] TextResponse.follow_all: Simplify implementation --- scrapy/http/response/text.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index b2907baa4..25e115bf9 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -191,14 +191,12 @@ class TextResponse(Response): if arg_count != 1: raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath') if not urls: - urls = [] if css: - selector_method = getattr(self, 'css') - expression = css - elif xpath: - selector_method = getattr(self, 'xpath') - expression = xpath - for selector in selector_method(expression): + selector_list = self.css(css) + if xpath: + selector_list = self.xpath(xpath) + urls = [] + for selector in selector_list: try: urls.append(_url_from_selector(selector)) except ValueError: From 2c6f7fee6456168f4870293c442258a2470b1b48 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Oct 2019 13:48:14 -0300 Subject: [PATCH 09/18] TextResponse.follow_all: invoke Response.follow_all --- scrapy/http/response/text.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 25e115bf9..f782f6217 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -201,22 +201,19 @@ class TextResponse(Response): urls.append(_url_from_selector(selector)) except ValueError: pass - return ( - self.follow( - url=url, - callback=callback, - method=method, - headers=headers, - body=body, - cookies=cookies, - meta=meta, - encoding=encoding, - priority=priority, - dont_filter=dont_filter, - errback=errback, - cb_kwargs=cb_kwargs, - ) - for url in urls + return super(TextResponse, self).follow_all( + urls=urls, + callback=callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback, + cb_kwargs=cb_kwargs, ) From 6df6b6dd6a5afd9172c03f6b9e8ce8e180867339 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 21 Oct 2019 03:56:45 -0300 Subject: [PATCH 10/18] Initializer -> __init__ --- scrapy/http/response/text.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f782f6217..74017b5aa 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -167,9 +167,9 @@ class TextResponse(Response): # type: (...) -> Generator[Request, None, None] """ A generator that produces :class:`~.Request` instances to follow all - links in ``urls``. It accepts the same arguments as the :class:`~.Request` - initializer, except that each ``urls`` element does not need to be an absolute - URL, it can be any of the following: + links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s + ``__init__`` method, except that each ``urls`` element does not need to be + an absolute URL, it can be any of the following: * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of From e6c5292a7c4391642897ee31ea2ded889b3b88dc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 09:29:55 -0300 Subject: [PATCH 11/18] Response.follow_all: Specific exception for invalid selectors --- scrapy/http/response/text.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 74017b5aa..5110b4bd4 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,17 +5,18 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin +from contextlib import suppress import parsel -from w3lib.encoding import html_to_unicode, resolve_encoding, \ - html_body_declared_encoding, http_content_type_encoding +import six +from six.moves.urllib.parse import urljoin +from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, + http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace from scrapy.http.response import Response -from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str +from scrapy.utils.response import get_base_url class TextResponse(Response): @@ -197,10 +198,8 @@ class TextResponse(Response): selector_list = self.xpath(xpath) urls = [] for selector in selector_list: - try: + with suppress(_InvalidSelector): urls.append(_url_from_selector(selector)) - except ValueError: - pass return super(TextResponse, self).follow_all( urls=urls, callback=callback, @@ -217,18 +216,24 @@ class TextResponse(Response): ) +class _InvalidSelector(ValueError): + """ + Raised when a URL cannot be obtained from a Selector + """ + + def _url_from_selector(sel): # type: (parsel.Selector) -> str if isinstance(sel.root, six.string_types): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): - raise ValueError("Unsupported selector: %s" % sel) + raise _InvalidSelector("Unsupported selector: %s" % sel) if sel.root.tag not in ('a', 'link'): - raise ValueError("Only and elements are supported; got <%s>" % - sel.root.tag) + raise _InvalidSelector("Only and elements are supported; got <%s>" % + sel.root.tag) href = sel.root.get('href') if href is None: - raise ValueError("<%s> element has no href attribute: %s" % - (sel.root.tag, sel)) + raise _InvalidSelector("<%s> element has no href attribute: %s" % + (sel.root.tag, sel)) return strip_html5_whitespace(href) From b602c61e1cc00e04ce435078644d5ea0c0249903 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 09:38:54 -0300 Subject: [PATCH 12/18] [Test] Rename outdated sample files --- .../{sgml_linkextractor.html => linkextractor.html} | 0 ..._linkextractor_no_href.html => linkextractor_no_href.html} | 0 tests/test_http_response.py | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/sample_data/link_extractor/{sgml_linkextractor.html => linkextractor.html} (100%) rename tests/sample_data/link_extractor/{sgml_linkextractor_no_href.html => linkextractor_no_href.html} (100%) diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html similarity index 100% rename from tests/sample_data/link_extractor/sgml_linkextractor.html rename to tests/sample_data/link_extractor/linkextractor.html diff --git a/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html similarity index 100% rename from tests/sample_data/link_extractor/sgml_linkextractor_no_href.html rename to tests/sample_data/link_extractor/linkextractor_no_href.html diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 6d3c5cb9d..0ae1612b5 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -250,12 +250,12 @@ class BaseResponseTest(unittest.TestCase): yield req def _links_response(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') + body = get_testdata('link_extractor', 'linkextractor.html') resp = self.response_class('http://example.com/index', body=body) return resp def _links_response_no_href(self): - body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html') + body = get_testdata('link_extractor', 'linkextractor_no_href.html') resp = self.response_class('http://example.com/index', body=body) return resp From 6f4e84ecf95feabe15fda0819bc428f8e0d4d340 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 09:55:15 -0300 Subject: [PATCH 13/18] PEP8 adjustments for scrapy.http.response module --- scrapy/http/response/__init__.py | 6 ++++-- scrapy/http/response/html.py | 1 + scrapy/http/response/text.py | 2 ++ scrapy/http/response/xml.py | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 79a8d0ca0..b9e638551 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,8 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ +from typing import Generator + from six.moves.urllib.parse import urljoin from scrapy.http.request import Request @@ -41,8 +43,8 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError('%s url must be str, got %s:' % (type(self).__name__, - type(url).__name__)) + raise TypeError('%s url must be str, got %s:' % + (type(self).__name__, type(url).__name__)) url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index bd3559fbb..7eed052c2 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class HtmlResponse(TextResponse): pass diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index b97420345..6acf1026f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,6 +6,7 @@ See documentation in docs/topics/request-response.rst """ from contextlib import suppress +from typing import Generator import parsel import six @@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace +from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 1df33fee5..abf474a2f 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class XmlResponse(TextResponse): pass From 6781d2f5b261305a4f6aa41a2b485a2e5cc7bc76 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 09:58:25 -0300 Subject: [PATCH 14/18] Update sample file references --- tests/test_linkextractors.py | 2 +- tests/test_linkextractors_deprecated.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 57ef1694a..0b94f937f 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -16,7 +16,7 @@ class Base: escapes_whitespace = False def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') + body = get_testdata('link_extractor', 'linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_urls_type(self): diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 1366971be..388ed6ad4 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -111,7 +111,7 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase): class HtmlParserLinkExtractorTestCase(unittest.TestCase): def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') + body = get_testdata('link_extractor', 'linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_extraction(self): @@ -183,7 +183,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase): # than it should be. def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') + body = get_testdata('link_extractor', 'linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_extraction(self): From dd12f5fdcd5ce2c63e9a5d3626031f5d93c1628e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Nov 2019 15:59:59 +0100 Subject: [PATCH 15/18] Use Response.follow_all in the documentation where appropiate --- docs/intro/tutorial.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6b15a5fbd..a2775e0bb 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -625,12 +625,12 @@ attribute automatically. So the code can be shortened further:: for a in response.css('li.next a'): yield response.follow(a, callback=self.parse) -.. note:: +To create multiple requests from an iterable, you can use +:meth:`response.follow_all ` instead:: + + links = response.css('li.next a') + yield from response.follow_all(links, callback=self.parse) - ``response.follow(response.css('li.next a'))`` is not valid because - ``response.css`` returns a list-like object with selectors for all results, - not a single selector. A ``for`` loop like in the example above, or - ``response.follow(response.css('li.next a')[0])`` is fine. More examples and patterns -------------------------- @@ -647,13 +647,11 @@ this time for scraping author information:: start_urls = ['http://quotes.toscrape.com/'] def parse(self, response): - # follow links to author pages - for href in response.css('.author + a::attr(href)'): - yield response.follow(href, self.parse_author) + author_page_links = response.css('.author + a') + yield from response.follow_all(author_links, self.parse_author) - # follow pagination links - for href in response.css('li.next a::attr(href)'): - yield response.follow(href, self.parse) + pagination_links = response.css('li.next a') + yield from response.follow_all(pagination_links, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -669,8 +667,10 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. -Here we're passing callbacks to ``response.follow`` as positional arguments -to make the code shorter; it also works for ``scrapy.Request``. +Here we're passing callbacks to +:meth:`response.follow_all ` as positional +arguments to make the code shorter; it also works for +:class:`~scrapy.http.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. From 5980b0f2840f51f8f9c7c3a266be93527999dd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Dec 2019 16:47:44 +0100 Subject: [PATCH 16/18] =?UTF-8?q?Don=E2=80=99t=20use=20follow=5Fall=20wher?= =?UTF-8?q?e=20a=20single=20item=20is=20expected=20(#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/tutorial.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a2775e0bb..2f97017fc 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -616,20 +616,19 @@ instance; you still have to yield this Request. You can also pass a selector to ``response.follow`` instead of a string; this selector should extract necessary attributes:: - for href in response.css('li.next a::attr(href)'): - yield response.follow(href, callback=self.parse) + href = response.css('li.next a::attr(href)')[0] + yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href attribute automatically. So the code can be shortened further:: - for a in response.css('li.next a'): - yield response.follow(a, callback=self.parse) + a = response.css('li.next a')[0] + yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use :meth:`response.follow_all ` instead:: - links = response.css('li.next a') - yield from response.follow_all(links, callback=self.parse) + yield from response.follow_all(response.css('a'), callback=self.parse) More examples and patterns From 2a9f5a0aefae83fc0b1dc161d117a265148ad1ef Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 3 Dec 2019 15:56:50 -0300 Subject: [PATCH 17/18] Skip invalid links when passing SelectorLists to Response.follow_all --- scrapy/http/response/text.py | 19 +++++++++++-------- tests/test_http_response.py | 12 ++++++++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6acf1026f..e3646b2d5 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -7,10 +7,10 @@ See documentation in docs/topics/request-response.rst from contextlib import suppress from typing import Generator +from urllib.parse import urljoin import parsel import six -from six.moves.urllib.parse import urljoin from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace @@ -183,22 +183,25 @@ class TextResponse(Response): In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). - Note that when using the ``css`` or ``xpath`` parameters, this method will not produce - requests for selectors from which links cannot be obtained (for instance, anchor tags - without an ``href`` attribute) + Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or + using the ``css`` or ``xpath`` parameters, this method will not produce requests for + selectors from which links cannot be obtained (for instance, anchor tags without an + ``href`` attribute) """ arg_count = len(list(filter(None, (urls, css, xpath)))) if arg_count != 1: raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath') if not urls: if css: - selector_list = self.css(css) + urls = self.css(css) if xpath: - selector_list = self.xpath(xpath) + urls = self.xpath(xpath) + if isinstance(urls, parsel.SelectorList): + selectors = urls urls = [] - for selector in selector_list: + for sel in selectors: with suppress(_InvalidSelector): - urls.append(_url_from_selector(selector)) + urls.append(_url_from_selector(sel)) return super(TextResponse, self).follow_all( urls=urls, callback=callback, diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 36ccdfa1f..ce13650ce 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -579,8 +579,10 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/page/4/', ] response = self._links_response_no_href() - extracted = [r.url for r in response.follow_all(css='.pagination a')] - self.assertEqual(expected, extracted) + extracted1 = [r.url for r in response.follow_all(css='.pagination a')] + self.assertEqual(expected, extracted1) + extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))] + self.assertEqual(expected, extracted2) def test_follow_all_xpath(self): expected = [ @@ -598,8 +600,10 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/page/4/', ] response = self._links_response_no_href() - extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')] - self.assertEqual(expected, extracted) + extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')] + self.assertEqual(expected, extracted1) + extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))] + self.assertEqual(expected, extracted2) def test_follow_all_too_many_arguments(self): response = self._links_response() From c75cf15b7a8293fd83f6e8b27abca2e50b0a04ed Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 22 Jan 2020 10:38:59 -0300 Subject: [PATCH 18/18] Update CSS selectors in tutorial --- docs/intro/tutorial.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4d8d717b..c9d00eb74 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -616,19 +616,24 @@ instance; you still have to yield this Request. You can also pass a selector to ``response.follow`` instead of a string; this selector should extract necessary attributes:: - href = response.css('li.next a::attr(href)')[0] - yield response.follow(href, callback=self.parse) + for href in response.css('ul.pager a::attr(href)'): + yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href attribute automatically. So the code can be shortened further:: - a = response.css('li.next a')[0] - yield response.follow(a, callback=self.parse) + for a in response.css('ul.pager a'): + yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use :meth:`response.follow_all ` instead:: - yield from response.follow_all(response.css('a'), callback=self.parse) + anchors = response.css('ul.pager a') + yield from response.follow_all(anchors, callback=self.parse) + +or, shortening it further:: + + yield from response.follow_all(css='ul.pager a', callback=self.parse) More examples and patterns