Merge pull request #2433 from redapple/wrong-spidermodules-warning

[MRG+1] Warn user instead of failing for wrong SPIDER_MODULES setting

scrapy/spiderloader.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+import traceback
+import warnings
 
 from zope.interface import implementer
 
@@ -18,15 +20,21 @@ class SpiderLoader(object):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
         self._load_all_spiders()
-            
+
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
         for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
+            try:
+                for module in walk_modules(name):
+                    self._load_spiders(module)
+            except ImportError as e:
+                msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+                       "Check SPIDER_MODULES setting".format(
+                            modname=name, tb=traceback.format_exc()))
+                warnings.warn(msg, RuntimeWarning)
 
     @classmethod
     def from_settings(cls, settings):

tests/test_spiderloader/__init__.py

@@ -1,6 +1,7 @@
 import sys
 import os
 import shutil
+import warnings
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):
         crawler = runner.create_crawler('spider1')
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
         self.assertEqual(crawler.spidercls.name, 'spider1')
+
+    def test_bad_spider_modules_warning(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+            settings = Settings({'SPIDER_MODULES': [module]})
+            spider_loader = SpiderLoader.from_settings(settings)
+            self.assertIn("Could not load spiders from module", str(w[0].message))
+
+            spiders = spider_loader.list()
+            self.assertEqual(spiders, [])