diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py new file mode 100644 index 000000000..c91735ca2 --- /dev/null +++ b/scrapy/contrib/spidermanager.py @@ -0,0 +1,109 @@ +""" +SpiderManager is the class which locates and manages all website-specific +spiders +""" + +import sys +import urlparse + +from twisted.plugin import getCache + +from scrapy.spider.models import ISpider +from scrapy import log +from scrapy.conf import settings +from scrapy.utils.url import url_is_from_spider +from scrapy.utils.misc import load_object + +class TwistedPluginSpiderManager(object): + """Spider locator and manager""" + + def __init__(self): + self.loaded = False + self.default_domain = None + self.force_domain = None + self.spider_modules = None + + def fromdomain(self, domain_name): + return self.asdict().get(domain_name) + + def fromurl(self, url): + if self.force_domain: + return self.asdict().get(self.force_domain) + domain = urlparse.urlparse(url).hostname + domain = str(domain).replace('www.', '') + if domain: + if domain in self.asdict(): # try first locating by domain + return self.asdict()[domain] + else: # else search spider by spider + plist = self.asdict().values() + for p in plist: + if url_is_from_spider(url, p): + return p + spider = self.asdict().get(self.default_domain) + if not spider: # create a custom spider + spiderclassname = settings.get('DEFAULT_SPIDER') + if spiderclassname: + spider = load_object(spiderclassname)(domain) + self.add_spider(spider) + + return spider + + def asdict(self): + return self._spiders + + def load(self, spider_modules=None): + if spider_modules is None: + spider_modules = settings.getlist('SPIDER_MODULES') + self.spider_modules = spider_modules + self._invaliddict = {} + self._spiders = {} + + modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] + for module in modules: + for spider in self._getspiders(ISpider, module): + self.add_spider(spider) + self.loaded = True + + def add_spider(self, spider): + try: + ISpider.validateInvariants(spider) + self._spiders[spider.domain_name] = spider + except Exception, e: + self._invaliddict[spider.domain_name] = spider + # we can't use the log module here because it may not be available yet + print "WARNING: Could not load spider %s: %s" % (spider, e) + + def reload(self, spider_modules=None, skip_domains=None): + """Reload spiders by trying to discover any spiders added under the + spiders module/packages, removes any spiders removed. + + If skip_domains is passed those spiders won't be reloaded. + """ + skip_domains = set(skip_domains or []) + modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] + for m in modules: + reload(m) + reloaded = 0 + pdict = self.asdict() + for domain, spider in pdict.iteritems(): + if not domain in skip_domains: + reload(sys.modules[spider.__module__]) + reloaded += 1 + self.load(spider_modules=spider_modules) # second call to update spider instances + log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG) + + def _getspiders(self, interface, package): + """This is an override of twisted.plugin.getPlugin, because we're + interested in catching exceptions thrown when loading spiders such as + KeyboardInterrupt + """ + try: + allDropins = getCache(package) + for dropin in allDropins.itervalues(): + for plugin in dropin.plugins: + adapted = interface(plugin, None) + if adapted is not None: + yield adapted + except KeyboardInterrupt: + sys.stderr.write("Interrupted while loading Scrapy spiders\n") + sys.exit(2)