Crawling rules: make link extractors optional

docs/topics/spiders.rst

@@ -374,12 +374,14 @@ CrawlSpider
 Crawling rules
 ~~~~~~~~~~~~~~
 
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
+.. autoclass:: Rule
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
    defines how links will be extracted from each crawled page. Each produced link will
    be used to generate a :class:`~scrapy.http.Request` object, which will contain the
    link's text in its ``meta`` dictionary (under the ``link_text`` key).
+   If omitted, a default link extractor created with no arguments will be used,
+   resulting in all links being extracted.
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with

scrapy/spiders/crawl.py

@@ -12,9 +12,10 @@ import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
-from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.python import get_func_args
+from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.python import get_func_args
+from scrapy.utils.spider import iterate_spider_output
 
 
 def _identity(request, response):
@@ -28,10 +29,13 @@ def _get_method(method, spider):
         return getattr(spider, method, None)
 
 
+_default_link_extractor = LinkExtractor()
+
+
 class Rule(object):
 
-    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
-        self.link_extractor = link_extractor
+    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
+        self.link_extractor = link_extractor or _default_link_extractor
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links

tests/test_spider.py

@@ -177,6 +177,26 @@ class CrawlSpiderTest(SpiderTest):
     </body></html>"""
     spider_class = CrawlSpider
 
+    def test_rule_without_link_extractor(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name = "test"
+            allowed_domains = ['example.org']
+            rules = (
+                Rule(),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+
     def test_process_links(self):
 
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)