From 35a7059636182d40de9f8b0080e19aa00fd5a19f Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 17:16:38 -0300 Subject: [PATCH] cleanup and refactor of parse & fetch commands * removed scrapy.utils.fetch * each command schedule requests and start scrapy engine * fetch command instance BaseSpider if given url does not match any spider or match more than one * parse command schedule url if one spider matches * parse and fetch doesn't support multiple urls as parameter * force spider behavior --spider moved from BaseCommand to only commands: fetch, parse, crawl --- scrapy/command/commands/crawl.py | 75 +++++++++++----------- scrapy/command/commands/fetch.py | 30 +++++++-- scrapy/command/commands/parse.py | 106 +++++++++++++++++++------------ scrapy/command/models.py | 2 - scrapy/utils/fetch.py | 20 ------ scrapy/utils/spider.py | 1 - 6 files changed, 128 insertions(+), 106 deletions(-) delete mode 100644 scrapy/utils/fetch.py diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index a1115ffbd..00f375df2 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -20,6 +20,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider when arguments are urls") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ help="don't follow links (for use with URLs only)") @@ -29,12 +31,41 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - if opts.spider: - spider = spiders.create(opts.spider) - else: - spider = None - # aggregate urls and domains + urls, domains = self._split_urls_and_domains(args) + for dom in domains: + scrapymanager.crawl_domain(dom) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + for url in urls: + scrapymanager.crawl_url(url, spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + else: + for name, urls in self._group_urls_by_spider(urls): + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) + + scrapymanager.start() + + def _group_urls_by_spider(self, urls): + spider_urls = defaultdict(list) + for url in urls: + spider_names = spiders.find_by_request(Request(url)) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + return spider_urls.items() + + def _split_urls_and_domains(self, args): urls = [] domains = [] for arg in args: @@ -42,36 +73,4 @@ class Command(ScrapyCommand): urls.append(arg) else: domains.append(arg) - - # schedule first domains - for dom in domains: - scrapymanager.crawl_domain(dom) - - # if forced spider schedule urls directly - if spider: - for url in urls: - scrapymanager.crawl_url(url, spider) - else: - # group urls by spider - spider_urls = defaultdict(list) - find_by_url = lambda url: spiders.find_by_request(Request(url)) - for url in urls: - spider_names = find_by_url(url) - if not spider_names: - log.msg('Could not find spider for url: %s' % url, - log.ERROR) - elif len(spider_names) > 1: - log.msg('More than one spider found for url: %s' % url, - log.ERROR) - else: - spider_urls[spider_names[0]].append(url) - - # schedule grouped urls with same spider - for name, urls in spider_urls.iteritems(): - # instance spider for each url-list - spider = spiders.create(name) - for url in urls: - scrapymanager.crawl_url(url, spider) - - # crawl just scheduled arguments without keeping idle - scrapymanager.start() + return urls, domains diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index bbc2efa9e..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -1,7 +1,11 @@ import pprint +from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager +from scrapy.http import Request +from scrapy.spider import BaseSpider, spiders +from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -19,17 +23,33 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", + help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") def run(self, args, opts): - if len(args) != 1: - print "One URL is required" - return + if len(args) != 1 or not is_url(args[0]): + return False + responses = [] # to collect downloaded responses + request = Request(args[0], callback=responses.append, dont_filter=True) - responses = fetch(args) + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg("Could not find spider: %s" % opts.spider, log.ERROR) + else: + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default')) + + scrapymanager.crawl_request(request, spider) + scrapymanager.start() + + # display response if responses: if opts.headers: pprint.pprint(responses[0].headers) else: print responses[0].body + diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 23bdd2ed5..d67a26d79 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -1,11 +1,15 @@ from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager from scrapy.http import Request from scrapy.item import BaseItem from scrapy.spider import spiders from scrapy.utils import display +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.url import is_url from scrapy import log +from collections import defaultdict + class Command(ScrapyCommand): requires_project = True @@ -18,6 +22,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider") parser.add_option("--nolinks", dest="nolinks", action="store_true", \ help="don't show extracted links") parser.add_option("--noitems", dest="noitems", action="store_true", \ @@ -37,25 +43,13 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider_names = spiders.find_by_request(response.request) - if not spider_names: - log.msg('Cannot find spider for url: %s' % response.url, - level=log.ERROR) - return (), () - elif len(spider_names) > 1: - log.msg('More than one spider found for url: %s' % response.url, - level=log.ERROR) - return (), () - else: - spider = spiders.create(spider_names[0]) - if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) return (), () - result = callback_fcn(response) + result = iterate_spider_output(callback_fcn(response)) links = [i for i in result if isinstance(i, Request)] items = [self.pipeline_process(i, spider, opts) for i in result if \ isinstance(i, BaseItem)] @@ -78,36 +72,68 @@ class Command(ScrapyCommand): display.pprint(list(links)) def run(self, args, opts): - if not args: - print "An URL is required" + if not len(args) == 1 or not is_url(args[0]): + return False + + request = Request(args[0]) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + return + else: + spider = scrapymanager._create_spider_for_request(request, \ + log_none=True, log_multiple=True) + + if not spider: return - for response in fetch(args): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for "%s"' % response.url) - continue + responses = [] # to collect downloaded responses + request = request.replace(callback=responses.append) - if self.callbacks: - for callback in self.callbacks: - items, links = self.run_callback(spider, response, callback, args, opts) - self.print_results(items, links, callback, opts) + scrapymanager.crawl_request(request, spider) + scrapymanager.start() - elif opts.rules: - rules = getattr(spider, 'rules', None) - if rules: - items, links = [], [] - for rule in rules: - if rule.callback and rule.link_extractor.matches(response.url): - items, links = self.run_callback(spider, response, rule.callback, args, opts) - self.print_results(items, links, rule.callback, opts) - break - else: - log.msg('No rules found for spider "%s", please specify a callback for parsing' \ - % spider.domain_name) - continue + if not responses: + log.msg('No response returned', log.ERROR, spider=spider) + return + # now process response + # - if callbacks defined then call each one print results + # - if --rules option given search for matching spider's rule + # - default print result using default 'parse' spider's callback + response = responses[0] + + if self.callbacks: + # apply each callback + for callback in self.callbacks: + items, links = self.run_callback(spider, response, + callback, args, opts) + self.print_results(items, links, callback, opts) + elif opts.rules: + # search for matching spider's rule + if hasattr(spider, 'rules') and spider.rules: + items, links = [], [] + for rule in spider.rules: + if rule.link_extractor.matches(response.url) \ + and rule.callback: + + items, links = self.run_callback(spider, + response, rule.callback, + args, opts) + self.print_results(items, links, + rule.callback, opts) + # first-match rule breaks rules loop + break else: - items, links = self.run_callback(spider, response, 'parse', args, opts) - self.print_results(items, links, 'parse', opts) + log.msg('No rules found for spider "%s", ' \ + 'please specify a callback for parsing' \ + % spider.domain_name, log.ERROR) + else: + # default callback 'parse' + items, links = self.run_callback(spider, response, + 'parse', args, opts) + self.print_results(items, links, 'parse', opts) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index 12eeef930..049365ebb 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -57,8 +57,6 @@ class ScrapyCommand(object): help="log level (default: %s)" % settings['LOGLEVEL']) group.add_option("--nolog", action="store_true", dest="nolog", \ help="disable logging completely") - group.add_option("--spider", dest="spider", default=None, \ - help="always use this spider when arguments are urls") group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ help="write python cProfile stats to FILE") group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \ diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py deleted file mode 100644 index a1eed0fd8..000000000 --- a/scrapy/utils/fetch.py +++ /dev/null @@ -1,20 +0,0 @@ -from scrapy.http import Request -from scrapy.core.manager import scrapymanager - -def fetch(urls): - """Fetch a list of urls and return a list of the downloaded Scrapy - Responses. - - This is a blocking function not suitable for calling from spiders. Instead, - it is indended to be called from outside the framework such as Scrapy - commands or standalone scripts. - """ - responses = [] - for url in urls: - req = Request(url, callback=responses.append, dont_filter=True) - # @@@ request will require a suitable spider. - # If not will not be schedule - scrapymanager.crawl_request(req) - scrapymanager.start() - return responses - diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 9307fc234..29432f0e9 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,4 +4,3 @@ from scrapy.utils.misc import arg_to_iter def iterate_spider_output(result): return [result] if isinstance(result, BaseItem) else arg_to_iter(result) -