RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Skip invalid links when passing SelectorLists to Response.follow_all

This commit is contained in:
Eugenio Lacuesta 2019-12-03 15:56:50 -03:00
parent 5980b0f284
commit 2a9f5a0aef
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scrapy/http/response/text.py
View File

@ -7,10 +7,10 @@ See documentation in docs/topics/request-response.rst
from contextlib import suppress
from typing import Generator
from urllib.parse import urljoin
import parsel
import six
from six.moves.urllib.parse import urljoin
from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
@ -183,22 +183,25 @@ class TextResponse(Response):
In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
requests for selectors from which links cannot be obtained (for instance, anchor tags
without an ``href`` attribute)
Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
using the ``css`` or ``xpath`` parameters, this method will not produce requests for
selectors from which links cannot be obtained (for instance, anchor tags without an
``href`` attribute)
"""
arg_count = len(list(filter(None, (urls, css, xpath))))
if arg_count != 1:
raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
if not urls:
if css:
selector_list = self.css(css)
urls = self.css(css)
if xpath:
selector_list = self.xpath(xpath)
urls = self.xpath(xpath)
if isinstance(urls, parsel.SelectorList):
selectors = urls
urls = []
for selector in selector_list:
for sel in selectors:
with suppress(_InvalidSelector):
urls.append(_url_from_selector(selector))
urls.append(_url_from_selector(sel))
return super(TextResponse, self).follow_all(
urls=urls,
callback=callback,

12
tests/test_http_response.py
View File

@ -579,8 +579,10 @@ class TextResponseTest(BaseResponseTest):
'http://example.com/page/4/',
]
response = self._links_response_no_href()
extracted = [r.url for r in response.follow_all(css='.pagination a')]
self.assertEqual(expected, extracted)
extracted1 = [r.url for r in response.follow_all(css='.pagination a')]
self.assertEqual(expected, extracted1)
extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))]
self.assertEqual(expected, extracted2)
def test_follow_all_xpath(self):
expected = [
@ -598,8 +600,10 @@ class TextResponseTest(BaseResponseTest):
'http://example.com/page/4/',
]
response = self._links_response_no_href()
extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
self.assertEqual(expected, extracted)
extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
self.assertEqual(expected, extracted1)
extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))]
self.assertEqual(expected, extracted2)
def test_follow_all_too_many_arguments(self):
response = self._links_response()