Allow to load a default spider when no spider was found for a given url. Also a generic spider

was added in contrib/spiders

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40587

scrapy/trunk/scrapy/conf/project_template/scrapy_settings.py

@@ -15,6 +15,9 @@ DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
 DOWNLOAD_TIMEOUT = 600
 
+# use this spider class as default when no spider was found for a given url
+#DEFAULT_SPIDER = 'scrapy.contrib.spiders.generic.GenericSpider'
+
 # uncomment if you want to add your own custom scrapy commands
 #COMMANDS_MODULE = '$project_name.commands'
 #COMMANDS_SETTINGS_MODULE = '$project_name.conf.commands'

scrapy/trunk/scrapy/contrib/spiders/generic.py

@@ -0,0 +1,20 @@
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.link.extractors import RegexLinkExtractor
+
+class GenericSpider(CrawlSpider):
+    """
+    This generic crawler crawls an entire site. It can also be used as a default crawler
+    (see setting "DEFAULT_SPIDER")
+    """
+    
+    def __init__(self, domain_name):
+        self.domain_name = domain_name
+        self.rules = (
+            Rule(RegexLinkExtractor(allow_domains=(domain_name,)), self.parse_note, follow=True),
+        )
+        super(GenericSpider, self).__init__()
+        
+    def parse_note(self, response):
+        pass
+
+# not a singleton

scrapy/trunk/scrapy/spider/manager.py

@@ -12,6 +12,7 @@ from scrapy.spider.models import ISpider
 from scrapy import log
 from scrapy.conf import settings
 from scrapy.utils.url import url_is_from_spider
+from scrapy.utils.misc import load_class
 
 class SpiderManager(object):
     """Spider locator and manager"""
@@ -49,7 +50,14 @@ class SpiderManager(object):
                 for p in plist:
                     if url_is_from_spider(url, p):
                         return p
-        return self._alldict.get(self.default_domain)
+        spider = self._alldict.get(self.default_domain)
+        if not spider:                          # create a custom spider
+            spiderclassname = settings.get('DEFAULT_SPIDER')
+            if spiderclassname:
+                spider = load_class(spiderclassname)(domain)
+                self.add_spider(spider)
+            
+        return spider
 
     def asdict(self, include_disabled=True):
         self._load_on_demand()
@@ -75,22 +83,25 @@ class SpiderManager(object):
         self._invaliddict = {}
         self._alldict = {}
         self._enableddict = {}
-        enabled_spiders = self._enabled_spiders()
+        self._enabled_spiders = self._enabled_spiders()
 
         modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules]
         for module in modules:
             for spider in self._getspiders(ISpider, module):
-                try:
-                    ISpider.validateInvariants(spider) 
-                    self._alldict[spider.domain_name] = spider
-                    if spider.domain_name in enabled_spiders:
-                        self._enableddict[spider.domain_name] = spider
-                except Exception, e:
-                    self._invaliddict[spider.domain_name] = spider
-                    # we can't use the log module here because it may not be available yet
-                    print "WARNING: Could not load spider %s: %s" % (spider, e)
+                self.add_spider(spider)
         self.loaded = True
 
+    def add_spider(self, spider):
+        try:
+            ISpider.validateInvariants(spider)
+            self._alldict[spider.domain_name] = spider
+            if spider.domain_name in self._enabled_spiders:
+                self._enableddict[spider.domain_name] = spider
+        except Exception, e:
+            self._invaliddict[spider.domain_name] = spider
+            # we can't use the log module here because it may not be available yet
+            print "WARNING: Could not load spider %s: %s" % (spider, e)
+
     def reload(self, skip_domains=None):
         """
         Reload all enabled spiders.