From 948e3cd00328f9a23410b3a7197975f56efadd16 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 8 Dec 2016 12:50:26 +0100
Subject: [PATCH] Warn user instead of failing for wrong SPIDER_MODULES setting

---
 scrapy/spiderloader.py              | 13 ++++++++++---
 tests/test_spiderloader/__init__.py | 12 ++++++++++++
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index fbf68cec4..265182329 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+import warnings
 
 from zope.interface import implementer
 
@@ -18,15 +19,21 @@ class SpiderLoader(object):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
         self._load_all_spiders()
-            
+
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
         for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
+            try:
+                for module in walk_modules(name):
+                    self._load_spiders(module)
+            except ImportError as e:
+                msg = ("Could not load spiders from module '{}'; "
+                       "Check SPIDER_MODULES setting "
+                       "(exception: {})".format(name, str(e)))
+                warnings.warn(msg, RuntimeWarning)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index fbd2c1669..b2ad93b3f 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,6 +1,7 @@
 import sys
 import os
 import shutil
+import warnings
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):
         crawler = runner.create_crawler('spider1')
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
         self.assertEqual(crawler.spidercls.name, 'spider1')
+
+    def test_bad_spider_modules_warning(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+            settings = Settings({'SPIDER_MODULES': [module]})
+            spider_loader = SpiderLoader.from_settings(settings)
+            self.assertIn("Could not load spiders from module", str(w[0].message))
+
+            spiders = spider_loader.list()
+            self.assertEqual(spiders, [])