RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4057 from elacuesta/response_follow_all 

Response.follow_all
This commit is contained in:
Mikhail Korobov 2020-01-23 02:15:24 +05:00 committed by GitHub
parent 50310fc0f9 c75cf15b7a
commit c0a7dfbc01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 319 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
docs/intro/tutorial.rst
View File

@ -616,21 +616,25 @@ instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
for href in response.css('li.next a::attr(href)'):
for href in response.css('ul.pager a::attr(href)'):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
for a in response.css('li.next a'):
for a in response.css('ul.pager a'):
yield response.follow(a, callback=self.parse)
.. note::
To create multiple requests from an iterable, you can use
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
anchors = response.css('ul.pager a')
yield from response.follow_all(anchors, callback=self.parse)
or, shortening it further::
yield from response.follow_all(css='ul.pager a', callback=self.parse)
``response.follow(response.css('li.next a'))`` is not valid because
``response.css`` returns a list-like object with selectors for all results,
not a single selector. A ``for`` loop like in the example above, or
``response.follow(response.css('li.next a')[0])`` is fine.
More examples and patterns
--------------------------
@ -647,13 +651,11 @@ this time for scraping author information::
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
# follow links to author pages
for href in response.css('.author + a::attr(href)'):
yield response.follow(href, self.parse_author)
author_page_links = response.css('.author + a')
yield from response.follow_all(author_links, self.parse_author)
# follow pagination links
for href in response.css('li.next a::attr(href)'):
yield response.follow(href, self.parse)
pagination_links = response.css('li.next a')
yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
@ -669,8 +671,10 @@ This spider will start from the main page, it will follow all the links to the
authors pages calling the ``parse_author`` callback for each of them, and also
the pagination links with the ``parse`` callback as we saw before.
Here we're passing callbacks to ``response.follow`` as positional arguments
to make the code shorter; it also works for ``scrapy.Request``.
Here we're passing callbacks to
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
arguments to make the code shorter; it also works for
:class:`~scrapy.http.Request`.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.

4
docs/topics/request-response.rst
View File

@ -701,6 +701,8 @@ Response objects
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
@ -790,6 +792,8 @@ TextResponse objects
.. automethod:: TextResponse.follow
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is

70
scrapy/http/response/__init__.py
View File

@ -4,14 +4,15 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from typing import Generator
from urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.exceptions import NotSupported
from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
from scrapy.http.common import obsolete_setter
from scrapy.exceptions import NotSupported
class Response(object_ref):
@ -41,8 +42,8 @@ class Response(object_ref):
if isinstance(url, str):
self._url = url
else:
raise TypeError('%s url must be str, got %s:' % (type(self).__name__,
type(url).__name__))
raise TypeError('%s url must be str, got %s:' %
(type(self).__name__, type(url).__name__))
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -123,14 +124,51 @@ class Response(object_ref):
elif url is None:
raise ValueError("url can't be None")
url = self.urljoin(url)
return Request(url, callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs)
return Request(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Generator[Request, None, None]
"""
Return an iterable of :class:`~.Request` instances to follow all links
in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
not only absolute URLs.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
if not hasattr(urls, '__iter__'):
raise TypeError("'urls' argument must be an iterable")
return (
self.follow(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
for url in urls
)

106
scrapy/http/response/text.py
View File

@ -5,17 +5,19 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
from contextlib import suppress
from typing import Generator
from urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
class TextResponse(Response):
@ -40,7 +42,7 @@ class TextResponse(Response):
if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
@ -86,8 +88,8 @@ class TextResponse(Response):
if self._cached_benc is None:
content_type = to_unicode(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
auto_detect_fun=self._auto_detect_fun,
default_encoding=self._DEFAULT_ENCODING)
auto_detect_fun=self._auto_detect_fun,
default_encoding=self._DEFAULT_ENCODING)
self._cached_benc = benc
self._cached_ubody = ubody
return self._cached_benc
@ -126,13 +128,14 @@ class TextResponse(Response):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
* a relative URL
* a :class:`~scrapy.link.Link` object, e.g. the result of
:ref:`topics-link-extractors`
* a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
``response.css('a.my_link')[0]``
* an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
``response.css('a::attr(href)')[0]`` or
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
``response.xpath('//img/@src')[0]``
See :ref:`response-follow-example` for usage examples.
"""
@ -141,7 +144,66 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(url, callback,
return super(TextResponse, self).follow(
url=url,
callback=callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback,
cb_kwargs=cb_kwargs,
)
def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None, cb_kwargs=None,
css=None, xpath=None):
# type: (...) -> Generator[Request, None, None]
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
``__init__`` method, except that each ``urls`` element does not need to be
an absolute URL, it can be any of the following:
* a relative URL
* a :class:`~scrapy.link.Link` object, e.g. the result of
:ref:`topics-link-extractors`
* a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
``response.css('a.my_link')[0]``
* an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
``response.css('a::attr(href)')[0]`` or
``response.xpath('//img/@src')[0]``
In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
using the ``css`` or ``xpath`` parameters, this method will not produce requests for
selectors from which links cannot be obtained (for instance, anchor tags without an
``href`` attribute)
"""
arg_count = len(list(filter(None, (urls, css, xpath))))
if arg_count != 1:
raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
if not urls:
if css:
urls = self.css(css)
if xpath:
urls = self.xpath(xpath)
if isinstance(urls, parsel.SelectorList):
selectors = urls
urls = []
for sel in selectors:
with suppress(_InvalidSelector):
urls.append(_url_from_selector(sel))
return super(TextResponse, self).follow_all(
urls=urls,
callback=callback,
method=method,
headers=headers,
body=body,
@ -155,18 +217,24 @@ class TextResponse(Response):
)
class _InvalidSelector(ValueError):
"""
Raised when a URL cannot be obtained from a Selector
"""
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise ValueError("Unsupported selector: %s" % sel)
raise _InvalidSelector("Unsupported selector: %s" % sel)
if sel.root.tag not in ('a', 'link'):
raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
sel.root.tag)
raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
sel.root.tag)
href = sel.root.get('href')
if href is None:
raise ValueError("<%s> element has no href attribute: %s" %
(sel.root.tag, sel))
raise _InvalidSelector("<%s> element has no href attribute: %s" %
(sel.root.tag, sel))
return strip_html5_whitespace(href)

0
tests/sample_data/link_extractor/sgml_linkextractor.html → tests/sample_data/link_extractor/linkextractor.html
View File

25
tests/sample_data/link_extractor/linkextractor_no_href.html Normal file
View File

@ -0,0 +1,25 @@
<html>
<head>
<base href='http://example.com' />
<title>Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method</title>
</head>
<body>
<div class="quote">
<span class="text">“The world as we have created it is a process of our
thinking. It cannot be changed without changing our thinking.”</span>
<span>
by <small class="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div id="pagination" class="pagination">
Tags:
<a href="/page/1/">Page 1</a>
<a>Current</a>
<a href="/page/3/">Page 3</a>
<a href="/page/4/">Page 4</a>
</div>
</div>
</body>
</html>

130
tests/test_http_response.py
View File

@ -141,6 +141,8 @@ class BaseResponseTest(unittest.TestCase):
r.css('body')
r.xpath('//body')
# Response.follow
def test_follow_url_absolute(self):
self._assert_followed_url('http://foo.example.com',
'http://foo.example.com')
@ -164,6 +166,72 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_whitespace_link(self):
self._assert_followed_url(Link('http://example.com/foo '),
'http://example.com/foo%20')
# Response.follow_all
def test_follow_all_absolute(self):
url_list = ['http://example.org', 'http://www.example.org',
'http://example.com', 'http://www.example.com']
self._assert_followed_all_urls(url_list, url_list)
def test_follow_all_relative(self):
relative = ['foo', 'bar', 'foo/bar', 'bar/foo']
absolute = [
'http://example.com/foo',
'http://example.com/bar',
'http://example.com/foo/bar',
'http://example.com/bar/foo',
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_links(self):
absolute = [
'http://example.com/foo',
'http://example.com/bar',
'http://example.com/foo/bar',
'http://example.com/bar/foo',
]
links = map(Link, absolute)
self._assert_followed_all_urls(links, absolute)
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
if self.response_class == Response:
with self.assertRaises(TypeError):
list(r.follow_all(urls=None))
with self.assertRaises(TypeError):
list(r.follow_all(urls=12345))
with self.assertRaises(ValueError):
list(r.follow_all(urls=[None]))
else:
with self.assertRaises(ValueError):
list(r.follow_all(urls=None))
with self.assertRaises(TypeError):
list(r.follow_all(urls=12345))
with self.assertRaises(ValueError):
list(r.follow_all(urls=[None]))
def test_follow_all_whitespace(self):
relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
absolute = [
'http://example.com/foo%20',
'http://example.com/bar%20',
'http://example.com/foo/bar%20',
'http://example.com/bar/foo%20',
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_whitespace_links(self):
absolute = [
'http://example.com/foo ',
'http://example.com/bar ',
'http://example.com/foo/bar ',
'http://example.com/bar/foo ',
]
links = map(Link, absolute)
expected = [u.replace(' ', '%20') for u in absolute]
self._assert_followed_all_urls(links, expected)
def _assert_followed_url(self, follow_obj, target_url, response=None):
if response is None:
response = self._links_response()
@ -171,8 +239,21 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(req.url, target_url)
return req
def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
for req, target in zip(followed, target_urls):
self.assertEqual(req.url, target)
yield req
def _links_response(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
body = get_testdata('link_extractor', 'linkextractor.html')
resp = self.response_class('http://example.com/index', body=body)
return resp
def _links_response_no_href(self):
body = get_testdata('link_extractor', 'linkextractor_no_href.html')
resp = self.response_class('http://example.com/index', body=body)
return resp
@ -481,6 +562,53 @@ class TextResponseTest(BaseResponseTest):
)
self.assertEqual(req.encoding, 'cp1251')
def test_follow_all_css(self):
expected = [
'http://example.com/sample3.html',
'http://example.com/innertag.html',
]
response = self._links_response()
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
self.assertEqual(expected, extracted)
def test_follow_all_css_skip_invalid(self):
expected = [
'http://example.com/page/1/',
'http://example.com/page/3/',
'http://example.com/page/4/',
]
response = self._links_response_no_href()
extracted1 = [r.url for r in response.follow_all(css='.pagination a')]
self.assertEqual(expected, extracted1)
extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))]
self.assertEqual(expected, extracted2)
def test_follow_all_xpath(self):
expected = [
'http://example.com/sample3.html',
'http://example.com/innertag.html',
]
response = self._links_response()
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
self.assertEqual(expected, [r.url for r in extracted])
def test_follow_all_xpath_skip_invalid(self):
expected = [
'http://example.com/page/1/',
'http://example.com/page/3/',
'http://example.com/page/4/',
]
response = self._links_response_no_href()
extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
self.assertEqual(expected, extracted1)
extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))]
self.assertEqual(expected, extracted2)
def test_follow_all_too_many_arguments(self):
response = self._links_response()
with self.assertRaises(ValueError):
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
class HtmlResponseTest(TextResponseTest):

2
tests/test_linkextractors.py
View File

@ -19,7 +19,7 @@ class Base:
escapes_whitespace = False
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
body = get_testdata('link_extractor', 'linkextractor.html')
self.response = HtmlResponse(url='http://example.com/index', body=body)
def test_urls_type(self):