Merge pull request #2422 from rolando-contrib/nested-spiders-modules

[MRG+1] DOC State explicitly that spiders are loaded recursively.

docs/topics/api.rst

@@ -171,7 +171,8 @@ SpiderLoader API
 
        This class method is used by Scrapy to create an instance of the class.
        It's called with the current project settings, and it loads the spiders
-       found in the modules of the :setting:`SPIDER_MODULES` setting.
+       found recursively in the modules of the :setting:`SPIDER_MODULES`
+       setting.
 
        :param settings: project settings
        :type settings: :class:`~scrapy.settings.Settings` instance

tests/test_spiderloader/__init__.py

@@ -9,6 +9,7 @@ from twisted.trial import unittest
 # ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
 import scrapy
+import tempfile
 from scrapy.interfaces import ISpiderLoader
 from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
@@ -22,8 +23,7 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
-        self.tmpdir = self.mktemp()
-        os.mkdir(self.tmpdir)
+        self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
@@ -40,7 +40,7 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(set(self.spider_loader.list()),
-            set(['spider1', 'spider2', 'spider3']))
+            set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_load(self):
         spider1 = self.spider_loader.load("spider1")

tests/test_spiderloader/test_spiders/nested/spider4.py

@@ -0,0 +1,9 @@
+from scrapy.spiders import Spider
+
+class Spider4(Spider):
+    name = "spider4"
+    allowed_domains = ['spider4.com']
+
+    @classmethod
+    def handles_request(cls, request):
+        return request.url == 'http://spider4.com/onlythis'