first cleanup of spider manager api

- removed asdict() and reload() methods
- added list() method
- removed default spider

docs/topics/settings.rst

@@ -339,17 +339,6 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`.
 
-.. setting:: DEFAULT_SPIDER
-
-DEFAULT_SPIDER
---------------
-
-Default: ``None``
-
-The default spider class that will be instantiated for URLs for which no
-specific spider is found. This class must have a constructor which receives as
-only parameter the domain name of the given URL.
-
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT

scrapy/command/commands/list.py

@@ -9,4 +9,4 @@ class Command(ScrapyCommand):
         return "List available spiders"
 
     def run(self, args, opts):
-        print "\n".join(spiders.asdict().keys())
+        print "\n".join(spiders.list())

scrapy/command/models.py

@@ -57,8 +57,6 @@ class ScrapyCommand(object):
             help="log level (default: %s)" % settings['LOGLEVEL'])
         group.add_option("--nolog", action="store_true", dest="nolog", \
             help="disable logging completely")
-        group.add_option("--default-spider", dest="default_spider", default=None, \
-            help="use this spider when arguments are urls and no spider is found")
         group.add_option("--spider", dest="spider", default=None, \
             help="always use this spider when arguments are urls")
         group.add_option("--profile", dest="profile", metavar="FILE", default=None, \
@@ -101,10 +99,6 @@ class ScrapyCommand(object):
         if opts.nolog:
             settings.overrides['LOG_ENABLED'] = False
 
-        if opts.default_spider:
-            from scrapy.spider import spiders
-            spiders.default_domain = opts.default_spider
-            
         if opts.spider:
             from scrapy.spider import spiders
             spiders.force_domain = opts.spider

scrapy/conf/default_settings.py

@@ -37,8 +37,6 @@ DEFAULT_REQUEST_HEADERS = {
     'Accept-Language': 'en',
 }
 
-DEFAULT_SPIDER = None
-
 DEPTH_LIMIT = 0
 DEPTH_STATS = True
 

scrapy/contrib/spidermanager.py

@@ -13,44 +13,35 @@ from scrapy.spider.models import ISpider
 from scrapy import log
 from scrapy.conf import settings
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.misc import load_object
 
 class TwistedPluginSpiderManager(object):
     """Spider locator and manager"""
 
     def __init__(self):
         self.loaded = False
-        self.default_domain = None
         self.force_domain = None
-        self.spider_modules = None
+        self._invaliddict = {}
+        self._spiders = {}
 
     def fromdomain(self, domain):
-        return self.asdict().get(domain)
+        return self._spiders.get(domain)
 
     def fromurl(self, url):
         if self.force_domain:
-            return self.asdict().get(self.force_domain)
+            return self._spiders.get(self.force_domain)
         domain = urlparse.urlparse(url).hostname
         domain = str(domain).replace('www.', '')
         if domain:
-            if domain in self.asdict():         # try first locating by domain
-                return self.asdict()[domain]
+            if domain in self._spiders:         # try first locating by domain
+                return self._spiders[domain]
             else:                               # else search spider by spider
-                plist = self.asdict().values()
+                plist = self._spiders.values()
                 for p in plist:
                     if url_is_from_spider(url, p):
                         return p
-        spider = self.asdict().get(self.default_domain)
-        if not spider:                          # create a custom spider
-            spiderclassname = settings.get('DEFAULT_SPIDER')
-            if spiderclassname:
-                spider = load_object(spiderclassname)(domain)
-                self.add_spider(spider)
-            
-        return spider
 
-    def asdict(self):
-        return self._spiders
+    def list(self):
+        return self._spiders.keys()
 
     def load(self, spider_modules=None):
         if spider_modules is None:
@@ -66,32 +57,8 @@ class TwistedPluginSpiderManager(object):
         self.loaded = True
 
     def add_spider(self, spider):
-        try:
-            ISpider.validateInvariants(spider)
-            self._spiders[spider.domain_name] = spider
-        except Exception, e:
-            self._invaliddict[spider.domain_name] = spider
-            # we can't use the log module here because it may not be available yet
-            print "WARNING: Could not load spider %s: %s" % (spider, e)
-
-    def reload(self, spider_modules=None, skip_domains=None):
-        """Reload spiders by trying to discover any spiders added under the
-        spiders module/packages, removes any spiders removed.
-
-        If skip_domains is passed those spiders won't be reloaded.
-        """
-        skip_domains = set(skip_domains or [])
-        modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules]
-        for m in modules:
-            reload(m)
-        reloaded = 0
-        pdict = self.asdict()
-        for domain, spider in pdict.iteritems():
-            if not domain in skip_domains:
-                reload(sys.modules[spider.__module__])
-                reloaded += 1
-        self.load(spider_modules=spider_modules)  # second call to update spider instances
-        log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG)
+        ISpider.validateInvariants(spider)
+        self._spiders[spider.domain_name] = spider
 
     def _getspiders(self, interface, package):
         """This is an override of twisted.plugin.getPlugin, because we're

scrapy/contrib/webconsole/spiderctl.py

@@ -35,7 +35,7 @@ class Spiderctl(object):
         if wc_request.args:
             changes = self.webconsole_control(wc_request)
 
-        enabled_domains = spiders.asdict().keys()
+        enabled_domains = spiders.list()
         self.scheduled = scrapyengine.domain_scheduler.pending_domains
         self.idle = [d for d in enabled_domains if d not in self.scheduled
                                                         and d not in self.running
@@ -139,7 +139,7 @@ class Spiderctl(object):
 
     def webconsole_control(self, wc_request):
         args = wc_request.args
-        enabled_domains = spiders.asdict().keys()
+        enabled_domains = spiders.list()
         s = "<hr />\n"
 
         if "stop_running_domains" in args:

scrapy/fetcher.py

@@ -43,7 +43,7 @@ def get_or_create_spider(url):
     # XXX: hack to allow downloading pages from unknown domains
     spider = spiders.fromurl(url)
     if not spider:
-        domain = str(urlparse.urlparse(url).hostname or spiders.default_domain)
+        domain = urlparse.urlparse(url).hostname
         spider = BaseSpider()
         spider.domain_name = domain
         spiders.add_spider(spider)