mirror of https://github.com/scrapy/scrapy.git
TextResponse.follow_all: skip invalid links
This commit is contained in:
Eugenio Lacuesta
parent
5f168cd459
commit
e1fa1fd8ad
|
|
@ -180,13 +180,27 @@ class TextResponse(Response):
|
|||
|
||||
In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
|
||||
within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
|
||||
|
||||
Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
|
||||
requests for selectors from which links cannot be obtained (for instance, anchor tags
|
||||
without ``href`` attribute)
|
||||
"""
|
||||
if len(list(filter(None, (urls, css, xpath)))) > 1:
|
||||
raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')
|
||||
if css:
|
||||
urls = self.css(css)
|
||||
elif xpath:
|
||||
urls = self.xpath(xpath)
|
||||
arg_count = len(list(filter(None, (urls, css, xpath))))
|
||||
if arg_count != 1:
|
||||
raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}')
|
||||
if not urls:
|
||||
urls = []
|
||||
if css:
|
||||
selector_method = getattr(self, 'css')
|
||||
expression = css
|
||||
elif xpath:
|
||||
selector_method = getattr(self, 'xpath')
|
||||
expression = xpath
|
||||
for selector in selector_method(expression):
|
||||
try:
|
||||
urls.append(_url_from_selector(selector))
|
||||
except ValueError:
|
||||
pass
|
||||
return (
|
||||
self.follow(
|
||||
url=url,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="quote">
|
||||
<span class="text">“The world as we have created it is a process of our
|
||||
thinking. It cannot be changed without changing our thinking.”</span>
|
||||
<span>
|
||||
by <small class="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div id="pagination" class="pagination">
|
||||
Tags:
|
||||
<a href="/page/1/">Page 1</a>
|
||||
<a>Current</a>
|
||||
<a href="/page/3/">Page 3</a>
|
||||
<a href="/page/4/">Page 4</a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -198,12 +198,20 @@ class BaseResponseTest(unittest.TestCase):
|
|||
|
||||
def test_follow_all_invalid(self):
|
||||
r = self.response_class("http://example.com")
|
||||
with self.assertRaises(TypeError):
|
||||
list(r.follow_all(None))
|
||||
with self.assertRaises(TypeError):
|
||||
list(r.follow_all(12345))
|
||||
with self.assertRaises(ValueError):
|
||||
list(r.follow_all([None]))
|
||||
if self.response_class == Response:
|
||||
with self.assertRaises(TypeError):
|
||||
list(r.follow_all(urls=None))
|
||||
with self.assertRaises(TypeError):
|
||||
list(r.follow_all(urls=12345))
|
||||
with self.assertRaises(ValueError):
|
||||
list(r.follow_all(urls=[None]))
|
||||
else:
|
||||
with self.assertRaises(ValueError):
|
||||
list(r.follow_all(urls=None))
|
||||
with self.assertRaises(TypeError):
|
||||
list(r.follow_all(urls=12345))
|
||||
with self.assertRaises(ValueError):
|
||||
list(r.follow_all(urls=[None]))
|
||||
|
||||
def test_follow_all_whitespace(self):
|
||||
relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
|
||||
|
|
@ -246,6 +254,11 @@ class BaseResponseTest(unittest.TestCase):
|
|||
resp = self.response_class('http://example.com/index', body=body)
|
||||
return resp
|
||||
|
||||
def _links_response_no_href(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html')
|
||||
resp = self.response_class('http://example.com/index', body=body)
|
||||
return resp
|
||||
|
||||
|
||||
class TextResponseTest(BaseResponseTest):
|
||||
|
||||
|
|
@ -560,6 +573,16 @@ class TextResponseTest(BaseResponseTest):
|
|||
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
|
||||
self.assertEqual(expected, extracted)
|
||||
|
||||
def test_follow_all_css_skip_invalid(self):
|
||||
expected = [
|
||||
'http://example.com/page/1/',
|
||||
'http://example.com/page/3/',
|
||||
'http://example.com/page/4/',
|
||||
]
|
||||
response = self._links_response_no_href()
|
||||
extracted = [r.url for r in response.follow_all(css='.pagination a')]
|
||||
self.assertEqual(expected, extracted)
|
||||
|
||||
def test_follow_all_xpath(self):
|
||||
expected = [
|
||||
'http://example.com/sample3.html',
|
||||
|
|
@ -569,7 +592,17 @@ class TextResponseTest(BaseResponseTest):
|
|||
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
|
||||
self.assertEqual(expected, [r.url for r in extracted])
|
||||
|
||||
def test_follow_all_exception(self):
|
||||
def test_follow_all_xpath_skip_invalid(self):
|
||||
expected = [
|
||||
'http://example.com/page/1/',
|
||||
'http://example.com/page/3/',
|
||||
'http://example.com/page/4/',
|
||||
]
|
||||
response = self._links_response_no_href()
|
||||
extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
|
||||
self.assertEqual(expected, extracted)
|
||||
|
||||
def test_follow_all_too_many_arguments(self):
|
||||
response = self._links_response()
|
||||
with self.assertRaises(ValueError):
|
||||
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
|
||||
|
|
|
|||
Loading…
Reference in New Issue