diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index f53de139d..a1115ffbd 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,8 +1,12 @@ +from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.http import Request +from scrapy.spider import spiders from scrapy.utils.url import is_url +from collections import defaultdict class Command(ScrapyCommand): @@ -25,12 +29,49 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): + if opts.spider: + spider = spiders.create(opts.spider) + else: + spider = None + + # aggregate urls and domains + urls = [] + domains = [] for arg in args: - # schedule arg as url or domain if is_url(arg): - scrapymanager.crawl_url(arg) + urls.append(arg) else: - scrapymanager.crawl_domain(arg) + domains.append(arg) + + # schedule first domains + for dom in domains: + scrapymanager.crawl_domain(dom) + + # if forced spider schedule urls directly + if spider: + for url in urls: + scrapymanager.crawl_url(url, spider) + else: + # group urls by spider + spider_urls = defaultdict(list) + find_by_url = lambda url: spiders.find_by_request(Request(url)) + for url in urls: + spider_names = find_by_url(url) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + + # schedule grouped urls with same spider + for name, urls in spider_urls.iteritems(): + # instance spider for each url-list + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) # crawl just scheduled arguments without keeping idle scrapymanager.start() diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f39d52d14..b2e0a5fb6 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -59,11 +59,17 @@ class Command(ScrapyCommand): module = sanitize_module_name(args[0]) domain = args[1] - spider = spiders.fromdomain(domain) - if spider and not opts.force: - print "Spider '%s' already exists in module:" % domain - print " %s" % spider.__module__ - sys.exit(1) + + # if spider already exists and not force option then halt + try: + spider = spiders.create(domain) + except KeyError: + pass + else: + if not opts.force: + print "Spider '%s' already exists in module:" % domain + print " %s" % spider.__module__ + sys.exit(1) template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 5d5143f08..23bdd2ed5 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -37,10 +37,17 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR) + spider_names = spiders.find_by_request(response.request) + if not spider_names: + log.msg('Cannot find spider for url: %s' % response.url, + level=log.ERROR) return (), () + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % response.url, + level=log.ERROR) + return (), () + else: + spider = spiders.create(spider_names[0]) if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index b019c3e52..12eeef930 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -99,10 +99,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - if opts.pidfile: with open(opts.pidfile, "w") as f: f.write(str(os.getpid())) diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 229301c2c..e37e861a5 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -19,35 +19,34 @@ class TwistedPluginSpiderManager(object): def __init__(self): self.loaded = False - self.force_domain = None - self._invaliddict = {} self._spiders = {} - def fromdomain(self, domain): - return self._spiders.get(domain) + def create(self, spider_id): + """ + Returns Spider instance by given identifier. + If not exists raises KeyError. + """ + #@@@ currently spider_id = domain + # if lookup fails let dict's KeyError exception propagate + return self._spiders[spider_id] - def fromurl(self, url): - if self.force_domain: - return self._spiders.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._spiders: # try first locating by domain - return self._spiders[domain] - else: # else search spider by spider - plist = self._spiders.values() - for p in plist: - if url_is_from_spider(url, p): - return p + def find_by_request(self, request): + """ + Returns list of spiders ids that match given Request. + """ + # just find by request.url + return [domain for domain, spider in self._spiders.iteritems() + if url_is_from_spider(request.url, spider)] def list(self): + """Returns list of spiders available.""" return self._spiders.keys() def load(self, spider_modules=None): + """Load spiders from module directory.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._invaliddict = {} self._spiders = {} modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 05586a117..dc6bba782 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -6,20 +6,13 @@ from scrapy.extension import extensions from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import spiders from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class ExecutionManager(object): - """Process a list of sites or urls. - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ def __init__(self): self.interrupted = False self.configured = False @@ -45,29 +38,30 @@ class ExecutionManager(object): def crawl_url(self, url, spider=None): """Schedule given url for crawling.""" - spider = spider or spiders.fromurl(url) + if spider is None: + spider = self._create_spider_for_request(Request(url), log_none=True, \ + log_multiple=True) if spider: requests = arg_to_iter(spider.make_requests_from_url(url)) self._crawl_requests(requests, spider) - else: - log.msg('Could not find spider for url: %s' % url, log.ERROR) def crawl_request(self, request, spider=None): """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider = spider or spiders.fromurl(request.url) + if spider is None: + spider = self._create_spider_for_request(request, log_none=True, \ + log_multiple=True) if spider: scrapyengine.crawl(request, spider) - else: - log.msg('Could not find spider for request: %s' % url, log.ERROR) def crawl_domain(self, domain): """Schedule given domain for crawling.""" - spider = spiders.fromdomain(domain) - if spider: - self.crawl_spider(spider) - else: + try: + spider = spiders.create(domain) + except KeyError: log.msg('Could not find spider for domain: %s' % domain, log.ERROR) + else: + self.crawl_spider(spider) def crawl_spider(self, spider): """Schedule spider for crawling.""" @@ -75,6 +69,7 @@ class ExecutionManager(object): self._crawl_requests(requests, spider) def _crawl_requests(self, requests, spider): + """Shortcut to schedule a list of requests""" for req in requests: self.crawl_request(req, spider) @@ -90,6 +85,17 @@ class ExecutionManager(object): self.interrupted = True scrapyengine.stop() + def _create_spider_for_request(self, request, default=None, log_none=False, \ + log_multiple=False): + spider_names = spiders.find_by_request(request) + if len(spider_names) == 1: + return spiders.create(spider_names[0]) + if len(spider_names) > 1 and log_multiple: + log.msg('More than one spider found for: %s' % request, log.ERROR) + if len(spider_names) == 0 and log_none: + log.msg('Could not find spider for: %s' % request, log.ERROR) + return default + def _signal_shutdown(self, signum, _): signame = signal_names[signum] log.msg("Received %s, shutting down gracefully. Send again to force " \ diff --git a/scrapy/shell.py b/scrapy/shell.py index 82baf63eb..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -35,6 +35,7 @@ def parse_url(url): u = urlparse.urlparse(url) return url + class Shell(object): requires_project = False @@ -52,18 +53,21 @@ class Shell(object): else: url = parse_url(request_or_url) request = Request(url) - spider = spiders.fromurl(url) or BaseSpider('default') + + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default'), log_multiple=True) + print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ request, spider) if response: - self.populate_vars(url, response, request) + self.populate_vars(url, response, request, spider) if print_help: self.print_help() else: print "Done - use shelp() to see available objects" - def populate_vars(self, url=None, response=None, request=None): + def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() self.vars['item'] = item if url: @@ -73,7 +77,7 @@ class Shell(object): self.vars['url'] = url self.vars['response'] = response self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) + self.vars['spider'] = spider if not self.nofetch: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser