From dd477914db1c4713dd1e73f4f5a8e9fed524361c Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 17:16:38 -0300 Subject: [PATCH] spidermanager refactoring * Implements find/create method in Spider Manager API, removed fromdomain and fromurl This method is now in charge of spider resolution, it must return spider object from its argument or raise KeyError if no spider is found. This method obsoletes from_domain and from_url methods. The default implementation of resolve only searches against spider.name, it won't use spider.allowed_domains like the old fromdomain. This is the reason of why you must supply a spider if you want to crawl an url. Find methods returns only available spider names. Not spider instances. If no spider found returns empty list. Affected modules: * command.models (force_domain) * removed spiders.force_domain * each command pass spider to crawl_* commands * command.commands.* * crawl * set spider from opts.spider if arg is url * group urls by spider to instance spider just once * genspider * use spiders.create() to check spider id * parse * log error if more than one spider found * core.manager * on crawl_* log message if multiple spiders found for url or request * shell * prints "Multiple found" if more than one spider found for url or request * populate_vars(): added spider keyword parameter * contrib.spidermanager: * removed fromdomain() & fromurl() * new create(spider_id) -> Spider. Raises KeyError if spider not found * new find_by_request(request) -> list(spiders) --- scrapy/command/commands/crawl.py | 47 ++++++++++++++++++++++++++-- scrapy/command/commands/genspider.py | 16 +++++++--- scrapy/command/commands/parse.py | 13 ++++++-- scrapy/command/models.py | 4 --- scrapy/contrib/spidermanager.py | 35 ++++++++++----------- scrapy/core/manager.py | 42 ++++++++++++++----------- scrapy/shell.py | 12 ++++--- 7 files changed, 114 insertions(+), 55 deletions(-) diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index f53de139d..a1115ffbd 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,8 +1,12 @@ +from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.http import Request +from scrapy.spider import spiders from scrapy.utils.url import is_url +from collections import defaultdict class Command(ScrapyCommand): @@ -25,12 +29,49 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): + if opts.spider: + spider = spiders.create(opts.spider) + else: + spider = None + + # aggregate urls and domains + urls = [] + domains = [] for arg in args: - # schedule arg as url or domain if is_url(arg): - scrapymanager.crawl_url(arg) + urls.append(arg) else: - scrapymanager.crawl_domain(arg) + domains.append(arg) + + # schedule first domains + for dom in domains: + scrapymanager.crawl_domain(dom) + + # if forced spider schedule urls directly + if spider: + for url in urls: + scrapymanager.crawl_url(url, spider) + else: + # group urls by spider + spider_urls = defaultdict(list) + find_by_url = lambda url: spiders.find_by_request(Request(url)) + for url in urls: + spider_names = find_by_url(url) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + + # schedule grouped urls with same spider + for name, urls in spider_urls.iteritems(): + # instance spider for each url-list + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) # crawl just scheduled arguments without keeping idle scrapymanager.start() diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f39d52d14..b2e0a5fb6 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -59,11 +59,17 @@ class Command(ScrapyCommand): module = sanitize_module_name(args[0]) domain = args[1] - spider = spiders.fromdomain(domain) - if spider and not opts.force: - print "Spider '%s' already exists in module:" % domain - print " %s" % spider.__module__ - sys.exit(1) + + # if spider already exists and not force option then halt + try: + spider = spiders.create(domain) + except KeyError: + pass + else: + if not opts.force: + print "Spider '%s' already exists in module:" % domain + print " %s" % spider.__module__ + sys.exit(1) template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 5d5143f08..23bdd2ed5 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -37,10 +37,17 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR) + spider_names = spiders.find_by_request(response.request) + if not spider_names: + log.msg('Cannot find spider for url: %s' % response.url, + level=log.ERROR) return (), () + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % response.url, + level=log.ERROR) + return (), () + else: + spider = spiders.create(spider_names[0]) if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index b019c3e52..12eeef930 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -99,10 +99,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - if opts.pidfile: with open(opts.pidfile, "w") as f: f.write(str(os.getpid())) diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 229301c2c..e37e861a5 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -19,35 +19,34 @@ class TwistedPluginSpiderManager(object): def __init__(self): self.loaded = False - self.force_domain = None - self._invaliddict = {} self._spiders = {} - def fromdomain(self, domain): - return self._spiders.get(domain) + def create(self, spider_id): + """ + Returns Spider instance by given identifier. + If not exists raises KeyError. + """ + #@@@ currently spider_id = domain + # if lookup fails let dict's KeyError exception propagate + return self._spiders[spider_id] - def fromurl(self, url): - if self.force_domain: - return self._spiders.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._spiders: # try first locating by domain - return self._spiders[domain] - else: # else search spider by spider - plist = self._spiders.values() - for p in plist: - if url_is_from_spider(url, p): - return p + def find_by_request(self, request): + """ + Returns list of spiders ids that match given Request. + """ + # just find by request.url + return [domain for domain, spider in self._spiders.iteritems() + if url_is_from_spider(request.url, spider)] def list(self): + """Returns list of spiders available.""" return self._spiders.keys() def load(self, spider_modules=None): + """Load spiders from module directory.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._invaliddict = {} self._spiders = {} modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 05586a117..dc6bba782 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -6,20 +6,13 @@ from scrapy.extension import extensions from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import spiders from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class ExecutionManager(object): - """Process a list of sites or urls. - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ def __init__(self): self.interrupted = False self.configured = False @@ -45,29 +38,30 @@ class ExecutionManager(object): def crawl_url(self, url, spider=None): """Schedule given url for crawling.""" - spider = spider or spiders.fromurl(url) + if spider is None: + spider = self._create_spider_for_request(Request(url), log_none=True, \ + log_multiple=True) if spider: requests = arg_to_iter(spider.make_requests_from_url(url)) self._crawl_requests(requests, spider) - else: - log.msg('Could not find spider for url: %s' % url, log.ERROR) def crawl_request(self, request, spider=None): """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider = spider or spiders.fromurl(request.url) + if spider is None: + spider = self._create_spider_for_request(request, log_none=True, \ + log_multiple=True) if spider: scrapyengine.crawl(request, spider) - else: - log.msg('Could not find spider for request: %s' % url, log.ERROR) def crawl_domain(self, domain): """Schedule given domain for crawling.""" - spider = spiders.fromdomain(domain) - if spider: - self.crawl_spider(spider) - else: + try: + spider = spiders.create(domain) + except KeyError: log.msg('Could not find spider for domain: %s' % domain, log.ERROR) + else: + self.crawl_spider(spider) def crawl_spider(self, spider): """Schedule spider for crawling.""" @@ -75,6 +69,7 @@ class ExecutionManager(object): self._crawl_requests(requests, spider) def _crawl_requests(self, requests, spider): + """Shortcut to schedule a list of requests""" for req in requests: self.crawl_request(req, spider) @@ -90,6 +85,17 @@ class ExecutionManager(object): self.interrupted = True scrapyengine.stop() + def _create_spider_for_request(self, request, default=None, log_none=False, \ + log_multiple=False): + spider_names = spiders.find_by_request(request) + if len(spider_names) == 1: + return spiders.create(spider_names[0]) + if len(spider_names) > 1 and log_multiple: + log.msg('More than one spider found for: %s' % request, log.ERROR) + if len(spider_names) == 0 and log_none: + log.msg('Could not find spider for: %s' % request, log.ERROR) + return default + def _signal_shutdown(self, signum, _): signame = signal_names[signum] log.msg("Received %s, shutting down gracefully. Send again to force " \ diff --git a/scrapy/shell.py b/scrapy/shell.py index 82baf63eb..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -35,6 +35,7 @@ def parse_url(url): u = urlparse.urlparse(url) return url + class Shell(object): requires_project = False @@ -52,18 +53,21 @@ class Shell(object): else: url = parse_url(request_or_url) request = Request(url) - spider = spiders.fromurl(url) or BaseSpider('default') + + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default'), log_multiple=True) + print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ request, spider) if response: - self.populate_vars(url, response, request) + self.populate_vars(url, response, request, spider) if print_help: self.print_help() else: print "Done - use shelp() to see available objects" - def populate_vars(self, url=None, response=None, request=None): + def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() self.vars['item'] = item if url: @@ -73,7 +77,7 @@ class Shell(object): self.vars['url'] = url self.vars['response'] = response self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) + self.vars['spider'] = spider if not self.nofetch: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser