diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0504a4e70..a79f1795e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -230,7 +230,7 @@ CrawlSpider Crawling rules ~~~~~~~~~~~~~~ -.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None) +.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None) ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. @@ -253,6 +253,10 @@ Crawling rules of links extracted from each response using the specified ``link_extractor``. This is mainly used for filtering purposes. + ``process_request`` is a callable, or a string (in which case a method from + the spider object with that name will be used) which will be called with + every request extracted by this rule, and must return a request or None (to + filter out the request). CrawlSpider example ------------------- diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 14d4fe40e..4ca606d8a 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -13,84 +13,40 @@ from scrapy.utils.spider import iterate_spider_output from scrapy.contrib.spiders.init import InitSpider from scrapy.conf import settings +def identity(x): + return x + class Rule(object): - """ - A rule for crawling, which receives the following constructor arguments: - link_extractor (required) - A LinkExtractor which defines the policy for extracting links - callback (optional) - A function to use to process the page once it has been downloaded. If - callback is omitted the page is not procesed, just crawled. If callback - is a string (instead of callable) a method of the spider class with that - name is used as the callback function - cb_kwargs (optional) - A dict specifying keyword arguments to pass to the callback function - follow (optional) - If True, links will be followed from the pages crawled by this rule. - It defaults to True when no callback is specified or False when a - callback is specified - process_links (optional) - Can be either a callable, or a string with the name of a method defined - in the spider's class. - This method will be called with the list of extracted links matching - this rule (if any) and must return another list of links. - """ - - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None): + def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): self.link_extractor = link_extractor self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links + self.process_request = process_request if follow is None: self.follow = False if callback else True else: self.follow = follow class CrawlSpider(InitSpider): - """ - Class for spiders that crawl over web pages and extract/parse their links - given some crawling rules. - These crawling rules are established by setting the 'rules' class attribute, - which is a tuple of Rule objects. - When the spider is running, it iterates over these rules with each response - and do what it has to (extract links if follow=True, and return items/requests if - there's a parsing method defined in the rule). - """ rules = () def __init__(self, *a, **kw): - """Constructor takes care of compiling rules""" super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() def parse(self, response): - """This function is called by the framework core for all the - start_urls. Do not override this function, override parse_start_url - instead.""" return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True) def parse_start_url(self, response): - """Overrideable callback function for processing start_urls. It must - return a list of BaseItem and/or Requests""" return [] def process_results(self, response, results): - """This overridable method is called for each result (item or request) - returned by the spider, and it's intended to perform any last time - processing required before returning the results to the framework core, - for example setting the item GUIDs. It receives a list of results and - the response which originated that results. It must return a list - of results (Items or Requests).""" return results def _requests_to_follow(self, response): - """ - This method iterates over each of the spider's rules, extracts the links - matching each case, filters them (if needed), and returns a list of unique - requests per response. - """ seen = set() for rule in self._rules: links = [l for l in rule.link_extractor.extract_links(response) if l not in seen] @@ -102,14 +58,9 @@ class CrawlSpider(InitSpider): cb_kwargs=rule.cb_kwargs, follow=rule.follow) r = Request(url=link.url, callback=callback) r.meta['link_text'] = link.text - yield r + yield rule.process_request(r) def _response_downloaded(self, response, callback, cb_kwargs, follow): - """ - This is were any response arrives, and were it's decided whether - to extract links or not from it, and if it will be parsed or not. - It returns a list of requests/items. - """ if callback: cb_res = callback(response, **cb_kwargs) or () cb_res = self.process_results(response, cb_res) @@ -122,8 +73,6 @@ class CrawlSpider(InitSpider): def _compile_rules(self): - """Compile the crawling rules""" - def get_method(method): if callable(method): return method @@ -134,3 +83,4 @@ class CrawlSpider(InitSpider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) + rule.process_request = get_method(rule.process_request)