Added support for unifying access to per-spider settings. Refs #245

scrapy/crawler.py

@@ -54,6 +54,7 @@ class Crawler(object):
     @defer.inlineCallbacks
     def _start_spider(self, spider, requests):
         """Don't call this method. Use self.queue to start new spiders"""
+        spider.set_crawler(self)
         yield defer.maybeDeferred(self.engine.open_spider, spider)
         for request in requests:
             self.engine.crawl(request, spider)

scrapy/settings/__init__.py

@@ -57,3 +57,25 @@ class CrawlerSettings(Settings):
 
     def __str__(self):
         return "<CrawlerSettings module=%r>" % self.settings_module.__name__
+
+
+class SpiderSettings(Settings):
+
+    def __init__(self, spider, crawler_settings, **kw):
+        super(SpiderSettings, self).__init__(**kw)
+        self.spider = spider
+        self.cset = crawler_settings
+
+    def __getitem__(self, opt_name):
+        if opt_name in self.cset.overrides:
+            return self.cset.overrides[opt_name]
+        if hasattr(self.spider, opt_name):
+            return getattr(self.spider, opt_name)
+        if self.cset.settings_module and hasattr(self.cset.settings_module, opt_name):
+            return getattr(self.cset.settings_module, opt_name)
+        if opt_name in self.cset.defaults:
+            return self.cset.defaults[opt_name]
+        return super(SpiderSettings, self).__getitem__(opt_name)
+
+    def __str__(self):
+        return "<SpiderSettings spider=%r>" % self.spider.name

scrapy/spider.py

@@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst
 """
 
 from scrapy import log
+from scrapy.settings import SpiderSettings
 from scrapy.http import Request
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.trackref import object_ref
@@ -33,6 +34,21 @@ class BaseSpider(object_ref):
         """
         log.msg(message, spider=self, level=level)
 
+    def set_crawler(self, crawler):
+        assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler
+        self._crawler = crawler
+
+    @property
+    def crawler(self):
+        assert hasattr(self, '_crawler'), "Spider not bounded to any crawler"
+        return self._crawler
+
+    @property
+    def settings(self):
+        if not hasattr(self, '_settings'):
+            self._settings = SpiderSettings(self, self.crawler.settings)
+        return self._settings
+
     def start_requests(self):
         reqs = []
         for url in self.start_urls: