Some changes to Crawl spider:

* added process_request attribute to rules
* removed docstrings, since it duplicates documentation

docs/topics/spiders.rst

@@ -230,7 +230,7 @@ CrawlSpider
        
 Crawling rules
 ~~~~~~~~~~~~~~
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None)
+.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
    defines how links will be extracted from each crawled page.
@@ -253,6 +253,10 @@ Crawling rules
    of links extracted from each response using the specified ``link_extractor``.
    This is mainly used for filtering purposes. 
 
+    ``process_request`` is a callable, or a string (in which case a method from
+    the spider object with that name will be used) which will be called with
+    every request extracted by this rule, and must return a request or None (to
+    filter out the request).
 
 CrawlSpider example
 -------------------

scrapy/contrib/spiders/crawl.py

@@ -13,84 +13,40 @@ from scrapy.utils.spider import iterate_spider_output
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.conf import settings
 
+def identity(x):
+    return x
+
 class Rule(object):
-    """
-    A rule for crawling, which receives the following constructor arguments:
 
-    link_extractor (required)
-       A LinkExtractor which defines the policy for extracting links
-    callback (optional)
-       A function to use to process the page once it has been downloaded. If
-       callback is omitted the page is not procesed, just crawled. If callback
-       is a string (instead of callable) a method of the spider class with that
-       name is used as the callback function
-    cb_kwargs (optional)
-       A dict specifying keyword arguments to pass to the callback function
-    follow (optional)
-       If True, links will be followed from the pages crawled by this rule.
-       It defaults to True when no callback is specified or False when a
-       callback is specified
-    process_links (optional)
-       Can be either a callable, or a string with the name of a method defined
-       in the spider's class.
-       This method will be called with the list of extracted links matching
-       this rule (if any) and must return another list of links.
-    """
-
-    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None):
+    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
         self.link_extractor = link_extractor
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
+        self.process_request = process_request
         if follow is None:
             self.follow = False if callback else True
         else:
             self.follow = follow
 
 class CrawlSpider(InitSpider):
-    """
-    Class for spiders that crawl over web pages and extract/parse their links
-    given some crawling rules.
 
-    These crawling rules are established by setting the 'rules' class attribute,
-    which is a tuple of Rule objects.
-    When the spider is running, it iterates over these rules with each response
-    and do what it has to (extract links if follow=True, and return items/requests if
-    there's a parsing method defined in the rule).
-    """
     rules = ()
 
     def __init__(self, *a, **kw):
-        """Constructor takes care of compiling rules"""
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
     def parse(self, response):
-        """This function is called by the framework core for all the
-        start_urls. Do not override this function, override parse_start_url
-        instead."""
         return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True)
 
     def parse_start_url(self, response):
-        """Overrideable callback function for processing start_urls. It must
-        return a list of BaseItem and/or Requests"""
         return []
 
     def process_results(self, response, results):
-        """This overridable method is called for each result (item or request)
-        returned by the spider, and it's intended to perform any last time
-        processing required before returning the results to the framework core,
-        for example setting the item GUIDs. It receives a list of results and
-        the response which originated that results. It must return a list
-        of results (Items or Requests)."""
         return results
 
     def _requests_to_follow(self, response):
-        """
-        This method iterates over each of the spider's rules, extracts the links
-        matching each case, filters them (if needed), and returns a list of unique
-        requests per response.
-        """
         seen = set()
         for rule in self._rules:
             links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
@@ -102,14 +58,9 @@ class CrawlSpider(InitSpider):
                     cb_kwargs=rule.cb_kwargs, follow=rule.follow)
                 r = Request(url=link.url, callback=callback)
                 r.meta['link_text'] = link.text
-                yield r
+                yield rule.process_request(r)
 
     def _response_downloaded(self, response, callback, cb_kwargs, follow):
-        """
-        This is were any response arrives, and were it's decided whether
-        to extract links or not from it, and if it will be parsed or not.
-        It returns a list of requests/items.
-        """
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
             cb_res = self.process_results(response, cb_res)
@@ -122,8 +73,6 @@ class CrawlSpider(InitSpider):
                 
 
     def _compile_rules(self):
-        """Compile the crawling rules"""
-
         def get_method(method):
             if callable(method):
                 return method
@@ -134,3 +83,4 @@ class CrawlSpider(InitSpider):
         for rule in self._rules:
             rule.callback = get_method(rule.callback)
             rule.process_links = get_method(rule.process_links)
+            rule.process_request = get_method(rule.process_request)