RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Response.follow_all

This commit is contained in:
Eugenio Lacuesta 2019-10-02 14:08:08 -03:00
parent fa56460d6a
commit 5f168cd459
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
4 changed files with 199 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/request-response.rst
View File

@ -701,6 +701,8 @@ Response objects
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
@ -790,6 +792,8 @@ TextResponse objects
.. automethod:: TextResponse.follow
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is

63
scrapy/http/response/__init__.py
View File

@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
@ -123,14 +123,51 @@ class Response(object_ref):
elif url is None:
raise ValueError("url can't be None")
url = self.urljoin(url)
return Request(url, callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs)
return Request(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Generator[Request, None, None]
"""
Return an iterable of :class:`~.Request` instance to follow all links
in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects
not only absolute URLs.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
if not hasattr(urls, '__iter__'):
raise TypeError("'urls' argument must be an iterable")
return (
self.follow(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
for url in urls
)

61
scrapy/http/response/text.py
View File

@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs, to_native_str
@ -44,7 +43,7 @@ class TextResponse(Response):
if isinstance(body, six.text_type):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
@ -90,8 +89,8 @@ class TextResponse(Response):
if self._cached_benc is None:
content_type = to_native_str(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
auto_detect_fun=self._auto_detect_fun,
default_encoding=self._DEFAULT_ENCODING)
auto_detect_fun=self._auto_detect_fun,
default_encoding=self._DEFAULT_ENCODING)
self._cached_benc = benc
self._cached_ubody = ubody
return self._cached_benc
@ -129,7 +128,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@ -137,7 +136,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
@ -145,7 +144,9 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(url, callback,
return super(TextResponse, self).follow(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
@ -158,6 +159,52 @@ class TextResponse(Response):
cb_kwargs=cb_kwargs,
)
def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None, cb_kwargs=None,
css=None, xpath=None):
# type: (...) -> Generator[Request, None, None]
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`
initializer, except that each ``urls`` element does not need to be an absolute
URL, it can be any of the following:
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
``response.css('a::attr(href)')[0]`` or
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
"""
if len(list(filter(None, (urls, css, xpath)))) > 1:
raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')
if css:
urls = self.css(css)
elif xpath:
urls = self.xpath(xpath)
return (
self.follow(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
for url in urls
)
def _url_from_selector(sel):
# type: (parsel.Selector) -> str

91
tests/test_http_response.py
View File

@ -143,6 +143,8 @@ class BaseResponseTest(unittest.TestCase):
r.css('body')
r.xpath('//body')
# Response.follow
def test_follow_url_absolute(self):
self._assert_followed_url('http://foo.example.com',
'http://foo.example.com')
@ -166,6 +168,64 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_whitespace_link(self):
self._assert_followed_url(Link('http://example.com/foo '),
'http://example.com/foo%20')
# Response.follow_all
def test_follow_all_absolute(self):
url_list = ['http://example.org', 'http://www.example.org',
'http://example.com', 'http://www.example.com']
self._assert_followed_all_urls(url_list, url_list)
def test_follow_all_relative(self):
relative = ['foo', 'bar', 'foo/bar', 'bar/foo']
absolute = [
'http://example.com/foo',
'http://example.com/bar',
'http://example.com/foo/bar',
'http://example.com/bar/foo',
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_links(self):
absolute = [
'http://example.com/foo',
'http://example.com/bar',
'http://example.com/foo/bar',
'http://example.com/bar/foo',
]
links = map(Link, absolute)
self._assert_followed_all_urls(links, absolute)
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
with self.assertRaises(TypeError):
list(r.follow_all(None))
with self.assertRaises(TypeError):
list(r.follow_all(12345))
with self.assertRaises(ValueError):
list(r.follow_all([None]))
def test_follow_all_whitespace(self):
relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
absolute = [
'http://example.com/foo%20',
'http://example.com/bar%20',
'http://example.com/foo/bar%20',
'http://example.com/bar/foo%20',
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_whitespace_links(self):
absolute = [
'http://example.com/foo ',
'http://example.com/bar ',
'http://example.com/foo/bar ',
'http://example.com/bar/foo ',
]
links = map(Link, absolute)
expected = [u.replace(' ', '%20') for u in absolute]
self._assert_followed_all_urls(links, expected)
def _assert_followed_url(self, follow_obj, target_url, response=None):
if response is None:
response = self._links_response()
@ -173,6 +233,14 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(req.url, target_url)
return req
def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
for req, target in zip(followed, target_urls):
self.assertEqual(req.url, target)
yield req
def _links_response(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
resp = self.response_class('http://example.com/index', body=body)
@ -483,6 +551,29 @@ class TextResponseTest(BaseResponseTest):
)
self.assertEqual(req.encoding, 'cp1251')
def test_follow_all_css(self):
expected = [
'http://example.com/sample3.html',
'http://example.com/innertag.html',
]
response = self._links_response()
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
self.assertEqual(expected, extracted)
def test_follow_all_xpath(self):
expected = [
'http://example.com/sample3.html',
'http://example.com/innertag.html',
]
response = self._links_response()
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
self.assertEqual(expected, [r.url for r in extracted])
def test_follow_all_exception(self):
response = self._links_response()
with self.assertRaises(ValueError):
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
class HtmlResponseTest(TextResponseTest):