diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 727c67482..9fe3c7518 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -701,6 +701,8 @@ Response objects
.. automethod:: Response.follow
+ .. automethod:: Response.follow_all
+
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
@@ -790,6 +792,8 @@ TextResponse objects
.. automethod:: TextResponse.follow
+ .. automethod:: TextResponse.follow_all
+
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b0a526b72..96359705e 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
@@ -123,14 +123,51 @@ class Response(object_ref):
elif url is None:
raise ValueError("url can't be None")
url = self.urljoin(url)
- return Request(url, callback,
- method=method,
- headers=headers,
- body=body,
- cookies=cookies,
- meta=meta,
- encoding=encoding,
- priority=priority,
- dont_filter=dont_filter,
- errback=errback,
- cb_kwargs=cb_kwargs)
+ return Request(
+ url=url,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+
+ def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
+ cookies=None, meta=None, encoding='utf-8', priority=0,
+ dont_filter=False, errback=None, cb_kwargs=None):
+ # type: (...) -> Generator[Request, None, None]
+ """
+ Return an iterable of :class:`~.Request` instance to follow all links
+ in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
+ but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects
+ not only absolute URLs.
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
+ method which supports selectors in addition to absolute/relative URLs
+ and Link objects.
+ """
+ if not hasattr(urls, '__iter__'):
+ raise TypeError("'urls' argument must be an iterable")
+ return (
+ self.follow(
+ url=url,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+ for url in urls
+ )
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 339913d4e..81e3f9b28 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
from w3lib.html import strip_html5_whitespace
-from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs, to_native_str
@@ -44,7 +43,7 @@ class TextResponse(Response):
if isinstance(body, six.text_type):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
- type(self).__name__)
+ type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
@@ -90,8 +89,8 @@ class TextResponse(Response):
if self._cached_benc is None:
content_type = to_native_str(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
- auto_detect_fun=self._auto_detect_fun,
- default_encoding=self._DEFAULT_ENCODING)
+ auto_detect_fun=self._auto_detect_fun,
+ default_encoding=self._DEFAULT_ENCODING)
self._cached_benc = benc
self._cached_ubody = ubody
return self._cached_benc
@@ -129,7 +128,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
-
+
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@@ -137,7 +136,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ```` or ```` element, e.g.
``response.css('a.my_link')[0]``.
-
+
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
@@ -145,7 +144,9 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
- return super(TextResponse, self).follow(url, callback,
+ return super(TextResponse, self).follow(
+ url=url,
+ callback=callback,
method=method,
headers=headers,
body=body,
@@ -158,6 +159,52 @@ class TextResponse(Response):
cb_kwargs=cb_kwargs,
)
+ def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
+ cookies=None, meta=None, encoding=None, priority=0,
+ dont_filter=False, errback=None, cb_kwargs=None,
+ css=None, xpath=None):
+ # type: (...) -> Generator[Request, None, None]
+ """
+ A generator that produces :class:`~.Request` instances to follow all
+ links in ``urls``. It accepts the same arguments as the :class:`~.Request`
+ initializer, except that each ``urls`` element does not need to be an absolute
+ URL, it can be any of the following:
+
+ * a relative URL;
+ * a scrapy.link.Link object (e.g. a link extractor result);
+ * an attribute Selector (not SelectorList) - e.g.
+ ``response.css('a::attr(href)')[0]`` or
+ ``response.xpath('//img/@src')[0]``.
+ * a Selector for ```` or ```` element, e.g.
+ ``response.css('a.my_link')[0]``.
+
+ In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
+ within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
+ """
+ if len(list(filter(None, (urls, css, xpath)))) > 1:
+ raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')
+ if css:
+ urls = self.css(css)
+ elif xpath:
+ urls = self.xpath(xpath)
+ return (
+ self.follow(
+ url=url,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+ for url in urls
+ )
+
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cd5c3486e..134856c78 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -143,6 +143,8 @@ class BaseResponseTest(unittest.TestCase):
r.css('body')
r.xpath('//body')
+ # Response.follow
+
def test_follow_url_absolute(self):
self._assert_followed_url('http://foo.example.com',
'http://foo.example.com')
@@ -166,6 +168,64 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_whitespace_link(self):
self._assert_followed_url(Link('http://example.com/foo '),
'http://example.com/foo%20')
+
+ # Response.follow_all
+
+ def test_follow_all_absolute(self):
+ url_list = ['http://example.org', 'http://www.example.org',
+ 'http://example.com', 'http://www.example.com']
+ self._assert_followed_all_urls(url_list, url_list)
+
+ def test_follow_all_relative(self):
+ relative = ['foo', 'bar', 'foo/bar', 'bar/foo']
+ absolute = [
+ 'http://example.com/foo',
+ 'http://example.com/bar',
+ 'http://example.com/foo/bar',
+ 'http://example.com/bar/foo',
+ ]
+ self._assert_followed_all_urls(relative, absolute)
+
+ def test_follow_all_links(self):
+ absolute = [
+ 'http://example.com/foo',
+ 'http://example.com/bar',
+ 'http://example.com/foo/bar',
+ 'http://example.com/bar/foo',
+ ]
+ links = map(Link, absolute)
+ self._assert_followed_all_urls(links, absolute)
+
+ def test_follow_all_invalid(self):
+ r = self.response_class("http://example.com")
+ with self.assertRaises(TypeError):
+ list(r.follow_all(None))
+ with self.assertRaises(TypeError):
+ list(r.follow_all(12345))
+ with self.assertRaises(ValueError):
+ list(r.follow_all([None]))
+
+ def test_follow_all_whitespace(self):
+ relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
+ absolute = [
+ 'http://example.com/foo%20',
+ 'http://example.com/bar%20',
+ 'http://example.com/foo/bar%20',
+ 'http://example.com/bar/foo%20',
+ ]
+ self._assert_followed_all_urls(relative, absolute)
+
+ def test_follow_all_whitespace_links(self):
+ absolute = [
+ 'http://example.com/foo ',
+ 'http://example.com/bar ',
+ 'http://example.com/foo/bar ',
+ 'http://example.com/bar/foo ',
+ ]
+ links = map(Link, absolute)
+ expected = [u.replace(' ', '%20') for u in absolute]
+ self._assert_followed_all_urls(links, expected)
+
def _assert_followed_url(self, follow_obj, target_url, response=None):
if response is None:
response = self._links_response()
@@ -173,6 +233,14 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(req.url, target_url)
return req
+ def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+ if response is None:
+ response = self._links_response()
+ followed = response.follow_all(follow_obj)
+ for req, target in zip(followed, target_urls):
+ self.assertEqual(req.url, target)
+ yield req
+
def _links_response(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
resp = self.response_class('http://example.com/index', body=body)
@@ -483,6 +551,29 @@ class TextResponseTest(BaseResponseTest):
)
self.assertEqual(req.encoding, 'cp1251')
+ def test_follow_all_css(self):
+ expected = [
+ 'http://example.com/sample3.html',
+ 'http://example.com/innertag.html',
+ ]
+ response = self._links_response()
+ extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
+ self.assertEqual(expected, extracted)
+
+ def test_follow_all_xpath(self):
+ expected = [
+ 'http://example.com/sample3.html',
+ 'http://example.com/innertag.html',
+ ]
+ response = self._links_response()
+ extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
+ self.assertEqual(expected, [r.url for r in extracted])
+
+ def test_follow_all_exception(self):
+ response = self._links_response()
+ with self.assertRaises(ValueError):
+ response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
+
class HtmlResponseTest(TextResponseTest):