RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2540 from scrapy/response-follow 

response.follow
This commit is contained in:
Daniel Graña 2017-02-20 11:21:21 -03:00 committed by GitHub
parent 377db31a32 fade5763af
commit c68140e68a
7 changed files with 277 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/intro/overview.rst
View File

@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::
next_page = response.css('li.next a::attr("href")').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``

73
docs/intro/tutorial.rst
View File

@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary::
>>>
Extracting data in our spider
------------------------------
-----------------------------
Let's get back to our spider. Until now, it doesn't extract any data in
particular, just saves the whole HTML page to a local file. Let's integrate the
@ -551,13 +551,65 @@ In our example, it creates a sort of loop, following all the links to the next p
until it doesn't find one -- handy for crawling blogs, forums and other sites with
pagination.
.. _response-follow-example:
A shortcut for creating Requests
--------------------------------
As a shortcut for creating Request objects you can use
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
import scrapy
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
]
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.css('span small::text').extract_first(),
'tags': quote.css('div.tags a.tag::text').extract(),
}
next_page = response.css('li.next a::attr(href)').extract_first()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
need to call urljoin. Note that ``response.follow`` just returns a Request
instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
for href in response.css('li.next a::attr(href)'):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
for a in response.css('li.next a'):
yield response.follow(a, callback=self.parse)
.. note::
``response.follow(response.css('li.next a'))`` is not valid because
``response.css`` returns a list-like object with selectors for all results,
not a single selector. A ``for`` loop like in the example above, or
``response.follow(response.css('li.next a')[0])`` is fine.
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
import scrapy
@ -568,15 +620,12 @@ this time for scraping author information::
def parse(self, response):
# follow links to author pages
for href in response.css('.author + a::attr(href)').extract():
yield scrapy.Request(response.urljoin(href),
callback=self.parse_author)
for href in response.css('.author + a::attr(href)'):
yield response.follow(href, self.parse_author)
# follow pagination links
next_page = response.css('li.next a::attr(href)').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
for href in response.css('li.next a::attr(href)'):
yield response.follow(href, self.parse)
def parse_author(self, response):
def extract_with_css(query):
@ -592,6 +641,9 @@ This spider will start from the main page, it will follow all the links to the
authors pages calling the ``parse_author`` callback for each of them, and also
the pagination links with the ``parse`` callback as we saw before.
Here we're passing callbacks to ``response.follow`` as positional arguments
to make the code shorter; it also works for ``scrapy.Request``.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@ -652,8 +704,7 @@ with a specific tag, building the URL based on the argument::
next_page = response.css('li.next a::attr(href)').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, self.parse)
yield response.follow(next_page, self.parse)
If you pass the ``tag=humor`` argument to this spider, you'll notice that it

5
docs/topics/request-response.rst
View File

@ -601,6 +601,9 @@ Response objects
urlparse.urljoin(response.url, url)
.. automethod:: Response.follow
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
.. _topics-request-response-ref-response-subclasses:
@ -687,6 +690,8 @@ TextResponse objects
response.css('p')
.. automethod:: TextResponse.follow
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is

30
scrapy/http/response/__init__.py
View File

@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst
"""
from six.moves.urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.http.headers import Headers
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
from scrapy.http.common import obsolete_setter
from scrapy.exceptions import NotSupported
@ -101,3 +103,31 @@ class Response(object_ref):
is text (subclasses of TextResponse).
"""
raise NotSupported("Response content isn't text")
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
dont_filter=False, errback=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
if isinstance(url, Link):
url = url.url
url = self.urljoin(url)
return Request(url, callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback)

56
scrapy/http/response/text.py
View File

@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst
import six
from six.moves.urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs, to_native_str
@ -116,3 +120,55 @@ class TextResponse(Response):
def css(self, query):
return self.selector.css(query)
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
``response.css('a::attr(href)')[0]`` or
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` element, e.g.
``response.css('a.my_link')[0]``.
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
url = _url_from_selector(url)
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(url, callback,
method=method,
headers=headers,
body=body,
cookies=cookies,
meta=meta,
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
errback=errback
)
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
if isinstance(sel.root, six.string_types):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise ValueError("Unsupported selector: %s" % sel)
if sel.root.tag != 'a':
raise ValueError("Only <a> elements are supported; got <%s>" %
sel.root.tag)
href = sel.root.get('href')
if href is None:
raise ValueError("<a> element has no href attribute: %s" % sel)
return strip_html5_whitespace(href)

7
tests/__init__.py
View File

@ -26,9 +26,12 @@ try:
except ImportError:
import mock
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'sample_data')
def get_testdata(*paths):
"""Return test data"""
path = os.path.join(tests_datadir, *paths)
return open(path, 'rb').read()
with open(path, 'rb') as f:
return f.read()

118
tests/test_http_response.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import unittest
import six
@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
from scrapy.selector import Selector
from scrapy.utils.python import to_native_str
from scrapy.exceptions import NotSupported
from scrapy.link import Link
from tests import get_testdata
class BaseResponseTest(unittest.TestCase):
@ -140,6 +143,38 @@ class BaseResponseTest(unittest.TestCase):
r.css('body')
r.xpath('//body')
def test_follow_url_absolute(self):
self._assert_followed_url('http://foo.example.com',
'http://foo.example.com')
def test_follow_url_relative(self):
self._assert_followed_url('foo',
'http://example.com/foo')
def test_follow_link(self):
self._assert_followed_url(Link('http://example.com/foo'),
'http://example.com/foo')
def test_follow_whitespace_url(self):
self._assert_followed_url('foo ',
'http://example.com/foo%20')
def test_follow_whitespace_link(self):
self._assert_followed_url(Link('http://example.com/foo '),
'http://example.com/foo%20')
def _assert_followed_url(self, follow_obj, target_url, response=None):
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
self.assertEqual(req.url, target_url)
return req
def _links_response(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
resp = self.response_class('http://example.com/index', body=body)
return resp
class TextResponseTest(BaseResponseTest):
@ -351,6 +386,89 @@ class TextResponseTest(BaseResponseTest):
absolute = 'http://www.example.com/elsewhere/test'
self.assertEqual(joined, absolute)
def test_follow_selector(self):
resp = self._links_response()
urls = [
'http://example.com/sample2.html',
'http://example.com/sample3.html',
'http://example.com/sample3.html',
'http://www.google.com/something',
'http://example.com/innertag.html'
]
# select <a> elements
for sellist in [resp.css('a'), resp.xpath('//a')]:
for sel, url in zip(sellist, urls):
self._assert_followed_url(sel, url, response=resp)
# href attributes should work
for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
for sel, url in zip(sellist, urls):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
def test_follow_selector_list(self):
resp = self._links_response()
self.assertRaisesRegexp(ValueError, 'SelectorList',
resp.follow, resp.css('a'))
def test_follow_selector_invalid(self):
resp = self._links_response()
self.assertRaisesRegexp(ValueError, 'Unsupported',
resp.follow, resp.xpath('count(//div)')[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
for src in resp.css('img::attr(src)'):
self._assert_followed_url(src, 'http://example.com/sample2.jpg')
def test_follow_selector_no_href(self):
resp = self.response_class(
url='http://example.com',
body=b'<html><body><a name=123>click me</a></body></html>',
)
self.assertRaisesRegexp(ValueError, 'no href',
resp.follow, resp.css('a')[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
'http://example.com',
body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
)
self._assert_followed_url(resp.css('a')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(resp.css('a::attr(href)')[0],
'http://example.com/foo',
response=resp)
def test_follow_encoding(self):
resp1 = self.response_class(
'http://example.com',
encoding='utf8',
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
)
req = self._assert_followed_url(
resp1.css('a')[0],
'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
response=resp1,
)
self.assertEqual(req.encoding, 'utf8')
resp2 = self.response_class(
'http://example.com',
encoding='cp1251',
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
)
req = self._assert_followed_url(
resp2.css('a')[0],
'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
response=resp2,
)
self.assertEqual(req.encoding, 'cp1251')
class HtmlResponseTest(TextResponseTest):