Merge pull request #1276 from scrapy/fix-spider-settings

Fix Spider.custom_settings

scrapy/crawler.py

@@ -29,6 +29,7 @@ class Crawler(object):
 
         self.spidercls = spidercls
         self.settings = settings.copy()
+        self.spidercls.update_settings(self.settings)
 
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
@@ -44,9 +45,7 @@ class Crawler(object):
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
-        self.spidercls.update_settings(self.settings)
         self.settings.freeze()
-
         self.crawling = False
         self.spider = None
         self.engine = None

tests/test_crawler.py

@@ -1,11 +1,13 @@
 import warnings
 import unittest
 
+import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.extensions.throttle import AutoThrottle
 
 
 class CrawlerTestCase(unittest.TestCase):
@@ -52,6 +54,18 @@ class CrawlerTestCase(unittest.TestCase):
         self.assertIsInstance(crawler.settings, Settings)
 
 
+class SpiderSettingsTestCase(unittest.TestCase):
+    def test_spider_custom_settings(self):
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'AUTOTHROTTLE_ENABLED': True
+            }
+
+        crawler = Crawler(MySpider, {})
+        enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+        self.assertIn(AutoThrottle, enabled_exts)
+
 
 class SpiderLoaderWithWrongInterface(object):