diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bf5fdfceb..35cce502a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -339,17 +339,6 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`. -.. setting:: DEFAULT_SPIDER - -DEFAULT_SPIDER --------------- - -Default: ``None`` - -The default spider class that will be instantiated for URLs for which no -specific spider is found. This class must have a constructor which receives as -only parameter the domain name of the given URL. - .. setting:: DEPTH_LIMIT DEPTH_LIMIT diff --git a/scrapy/command/commands/list.py b/scrapy/command/commands/list.py index f93ad0d4c..e1c4f440f 100644 --- a/scrapy/command/commands/list.py +++ b/scrapy/command/commands/list.py @@ -9,4 +9,4 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - print "\n".join(spiders.asdict().keys()) + print "\n".join(spiders.list()) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index 99d43c06b..b019c3e52 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -57,8 +57,6 @@ class ScrapyCommand(object): help="log level (default: %s)" % settings['LOGLEVEL']) group.add_option("--nolog", action="store_true", dest="nolog", \ help="disable logging completely") - group.add_option("--default-spider", dest="default_spider", default=None, \ - help="use this spider when arguments are urls and no spider is found") group.add_option("--spider", dest="spider", default=None, \ help="always use this spider when arguments are urls") group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ @@ -101,10 +99,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.default_spider: - from scrapy.spider import spiders - spiders.default_domain = opts.default_spider - if opts.spider: from scrapy.spider import spiders spiders.force_domain = opts.spider diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index ce1712fb2..02d2e712b 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -37,8 +37,6 @@ DEFAULT_REQUEST_HEADERS = { 'Accept-Language': 'en', } -DEFAULT_SPIDER = None - DEPTH_LIMIT = 0 DEPTH_STATS = True diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 349e878fe..b09debe89 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -13,44 +13,35 @@ from scrapy.spider.models import ISpider from scrapy import log from scrapy.conf import settings from scrapy.utils.url import url_is_from_spider -from scrapy.utils.misc import load_object class TwistedPluginSpiderManager(object): """Spider locator and manager""" def __init__(self): self.loaded = False - self.default_domain = None self.force_domain = None - self.spider_modules = None + self._invaliddict = {} + self._spiders = {} def fromdomain(self, domain): - return self.asdict().get(domain) + return self._spiders.get(domain) def fromurl(self, url): if self.force_domain: - return self.asdict().get(self.force_domain) + return self._spiders.get(self.force_domain) domain = urlparse.urlparse(url).hostname domain = str(domain).replace('www.', '') if domain: - if domain in self.asdict(): # try first locating by domain - return self.asdict()[domain] + if domain in self._spiders: # try first locating by domain + return self._spiders[domain] else: # else search spider by spider - plist = self.asdict().values() + plist = self._spiders.values() for p in plist: if url_is_from_spider(url, p): return p - spider = self.asdict().get(self.default_domain) - if not spider: # create a custom spider - spiderclassname = settings.get('DEFAULT_SPIDER') - if spiderclassname: - spider = load_object(spiderclassname)(domain) - self.add_spider(spider) - - return spider - def asdict(self): - return self._spiders + def list(self): + return self._spiders.keys() def load(self, spider_modules=None): if spider_modules is None: @@ -66,32 +57,8 @@ class TwistedPluginSpiderManager(object): self.loaded = True def add_spider(self, spider): - try: - ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider - except Exception, e: - self._invaliddict[spider.domain_name] = spider - # we can't use the log module here because it may not be available yet - print "WARNING: Could not load spider %s: %s" % (spider, e) - - def reload(self, spider_modules=None, skip_domains=None): - """Reload spiders by trying to discover any spiders added under the - spiders module/packages, removes any spiders removed. - - If skip_domains is passed those spiders won't be reloaded. - """ - skip_domains = set(skip_domains or []) - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for m in modules: - reload(m) - reloaded = 0 - pdict = self.asdict() - for domain, spider in pdict.iteritems(): - if not domain in skip_domains: - reload(sys.modules[spider.__module__]) - reloaded += 1 - self.load(spider_modules=spider_modules) # second call to update spider instances - log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG) + ISpider.validateInvariants(spider) + self._spiders[spider.domain_name] = spider def _getspiders(self, interface, package): """This is an override of twisted.plugin.getPlugin, because we're diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 8a167484f..853479179 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -35,7 +35,7 @@ class Spiderctl(object): if wc_request.args: changes = self.webconsole_control(wc_request) - enabled_domains = spiders.asdict().keys() + enabled_domains = spiders.list() self.scheduled = scrapyengine.domain_scheduler.pending_domains self.idle = [d for d in enabled_domains if d not in self.scheduled and d not in self.running @@ -139,7 +139,7 @@ class Spiderctl(object): def webconsole_control(self, wc_request): args = wc_request.args - enabled_domains = spiders.asdict().keys() + enabled_domains = spiders.list() s = "
\n" if "stop_running_domains" in args: diff --git a/scrapy/fetcher.py b/scrapy/fetcher.py index a9ae21b81..ee1498a34 100644 --- a/scrapy/fetcher.py +++ b/scrapy/fetcher.py @@ -43,7 +43,7 @@ def get_or_create_spider(url): # XXX: hack to allow downloading pages from unknown domains spider = spiders.fromurl(url) if not spider: - domain = str(urlparse.urlparse(url).hostname or spiders.default_domain) + domain = urlparse.urlparse(url).hostname spider = BaseSpider() spider.domain_name = domain spiders.add_spider(spider)