diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7b153979e..22d0f2f91 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -54,6 +54,7 @@ class Crawler(object): @defer.inlineCallbacks def _start_spider(self, spider, requests): """Don't call this method. Use self.queue to start new spiders""" + spider.set_crawler(self) yield defer.maybeDeferred(self.engine.open_spider, spider) for request in requests: self.engine.crawl(request, spider) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 30b80a1cf..7ca18b3ca 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -57,3 +57,25 @@ class CrawlerSettings(Settings): def __str__(self): return "" % self.settings_module.__name__ + + +class SpiderSettings(Settings): + + def __init__(self, spider, crawler_settings, **kw): + super(SpiderSettings, self).__init__(**kw) + self.spider = spider + self.cset = crawler_settings + + def __getitem__(self, opt_name): + if opt_name in self.cset.overrides: + return self.cset.overrides[opt_name] + if hasattr(self.spider, opt_name): + return getattr(self.spider, opt_name) + if self.cset.settings_module and hasattr(self.cset.settings_module, opt_name): + return getattr(self.cset.settings_module, opt_name) + if opt_name in self.cset.defaults: + return self.cset.defaults[opt_name] + return super(SpiderSettings, self).__getitem__(opt_name) + + def __str__(self): + return "" % self.spider.name diff --git a/scrapy/spider.py b/scrapy/spider.py index b2d2f0e46..cb090e3db 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst """ from scrapy import log +from scrapy.settings import SpiderSettings from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref @@ -33,6 +34,21 @@ class BaseSpider(object_ref): """ log.msg(message, spider=self, level=level) + def set_crawler(self, crawler): + assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler + self._crawler = crawler + + @property + def crawler(self): + assert hasattr(self, '_crawler'), "Spider not bounded to any crawler" + return self._crawler + + @property + def settings(self): + if not hasattr(self, '_settings'): + self._settings = SpiderSettings(self, self.crawler.settings) + return self._settings + def start_requests(self): reqs = [] for url in self.start_urls: