From 743a0aa422ae2f515cbea69153e60a02c39da98d Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Thu, 8 Sep 2016 21:52:14 +0200 Subject: [PATCH] Two fixes for when using the parse command and the '-r' flag (rules). 1. Use default "parse" as callback when the matching rule has no callback. 2. Log error and return when no rule matches the parsed url. --- scrapy/commands/parse.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6a8978415..5264982b6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -121,8 +121,8 @@ class Command(ScrapyCommand): def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): for rule in spider.rules: - if rule.link_extractor.matches(response.url) and rule.callback: - return rule.callback + if rule.link_extractor.matches(response.url): + return rule.callback or "parse" else: logger.error('No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', @@ -166,6 +166,11 @@ class Command(ScrapyCommand): if not cb: if opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) + + if not cb: + logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', + {'url': response.url, 'spider': spider.name}) + return else: cb = 'parse'