diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 869a61441..45eea3e60 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -374,12 +374,14 @@ CrawlSpider Crawling rules ~~~~~~~~~~~~~~ -.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None) +.. autoclass:: Rule ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will be used to generate a :class:`~scrapy.http.Request` object, which will contain the link's text in its ``meta`` dictionary (under the ``link_text`` key). + If omitted, a default link extractor created with no arguments will be used, + resulting in all links being extracted. ``callback`` is a callable or a string (in which case a method from the spider object with that name will be used) to be called for each link extracted with diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 90a6eb806..03000ce54 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -12,9 +12,10 @@ import six from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse -from scrapy.utils.spider import iterate_spider_output -from scrapy.utils.python import get_func_args +from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.python import get_func_args +from scrapy.utils.spider import iterate_spider_output def _identity(request, response): @@ -28,10 +29,13 @@ def _get_method(method, spider): return getattr(spider, method, None) +_default_link_extractor = LinkExtractor() + + class Rule(object): - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): - self.link_extractor = link_extractor + def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): + self.link_extractor = link_extractor or _default_link_extractor self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links diff --git a/tests/test_spider.py b/tests/test_spider.py index e81e6d5f9..2220b8ffc 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -177,6 +177,26 @@ class CrawlSpiderTest(SpiderTest): """ spider_class = CrawlSpider + def test_rule_without_link_extractor(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ['example.org'] + rules = ( + Rule(), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) + def test_process_links(self): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)