rename scrapy.spidermanager.SpiderManager to scrapy.spiderloader.SpiderLoader

docs/topics/api.rst

@@ -344,22 +344,22 @@ Settings API
 
        Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
 
-.. _topics-api-spidermanager:
+.. _topics-api-spiderloader:
 
-SpiderManager API
-=================
+SpiderLoader API
+================
 
-.. module:: scrapy.spidermanager
-   :synopsis: The spider manager
+.. module:: scrapy.loader
+   :synopsis: The spider loader
 
-.. class:: SpiderManager
+.. class:: SpiderLoader
 
     This class is in charge of retrieving and handling the spider classes
     defined across the project.
 
-    Custom spider managers can be employed by specifying their path in the
-    :setting:`SPIDER_MANAGER_CLASS` project setting. They must fully implement
-    the :class:`scrapy.interfaces.ISpiderManager` interface to guarantee an
+    Custom spider loaders can be employed by specifying their path in the
+    :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
+    the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
     errorless execution.
 
     .. method:: from_settings(settings)
@@ -486,7 +486,7 @@ class (which they all inherit from).
 
         Set the given value for the given key only if current value for the
         same key is lower than value. If there is no current value for the
-        given key, the value is always set. 
+        given key, the value is always set.
 
     .. method:: min_value(key, value)
 

docs/topics/settings.rst

@@ -853,15 +853,15 @@ A dict containing the scrapy contracts enabled by default in Scrapy. You should
 never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
 instead. For more info see :ref:`topics-contracts`.
 
-.. setting:: SPIDER_MANAGER_CLASS
+.. setting:: SPIDER_LOADER_CLASS
 
-SPIDER_MANAGER_CLASS
---------------------
+SPIDER_LOADER_CLASS
+-------------------
 
-Default: ``'scrapy.spidermanager.SpiderManager'``
+Default: ``'scrapy.spiderloader.SpiderLoader'``
 
-The class that will be used for handling spiders, which must implement the
-:ref:`topics-api-spidermanager`.
+The class that will be used for loading spiders, which must implement the
+:ref:`topics-api-spiderloader`.
 
 .. setting:: SPIDER_MIDDLEWARES
 

scrapy/crawler.py

@@ -7,7 +7,7 @@ from zope.interface.verify import verifyClass
 
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderManager
+from scrapy.interfaces import ISpiderLoader
 from scrapy.extension import ExtensionManager
 from scrapy.settings import Settings
 from scrapy.signalmanager import SignalManager
@@ -44,11 +44,10 @@ class Crawler(object):
         if not hasattr(self, '_spiders'):
             warnings.warn("Crawler.spiders is deprecated, use "
                           "CrawlerRunner.spiders or instantiate "
-                          "scrapy.spidermanager.SpiderManager with your "
+                          "scrapy.spiderloader.SpiderLoader with your "
                           "settings.",
                           category=ScrapyDeprecationWarning, stacklevel=2)
-            spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-            self._spiders = spman_cls.from_settings(self.settings)
+            self._spiders = _get_spider_loader(self.settings.frozencopy())
         return self._spiders
 
     @defer.inlineCallbacks
@@ -85,9 +84,7 @@ class CrawlerRunner(object):
         if isinstance(settings, dict):
             settings = Settings(settings)
         self.settings = settings
-        smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
-        verifyClass(ISpiderManager, smcls)
-        self.spiders = smcls.from_settings(settings.frozencopy())
+        self.spiders = _get_spider_loader(settings)
         self.crawlers = set()
         self._active = set()
 
@@ -178,3 +175,18 @@ class CrawlerProcess(CrawlerRunner):
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
+
+
+def _get_spider_loader(settings):
+    """ Get SpiderLoader instance from settings """
+    if settings.get('SPIDER_MANAGER_CLASS'):
+        warnings.warn(
+            'SPIDER_MANAGER_CLASS option is deprecated. '
+            'Please use SPIDER_LOADER_CLASS.',
+            category=ScrapyDeprecationWarning, stacklevel=2
+        )
+    cls_path = settings.get('SPIDER_LOADER_CLASS',
+                            settings.get('SPIDER_MANAGER_CLASS'))
+    loader_cls = load_object(cls_path)
+    verifyClass(ISpiderLoader, loader_cls)
+    return loader_cls.from_settings(settings.frozencopy())

scrapy/interfaces.py

@@ -1,12 +1,12 @@
 from zope.interface import Interface
 
-class ISpiderManager(Interface):
+class ISpiderLoader(Interface):
 
     def from_settings(settings):
-        """Returns an instance of the class for the given settings"""
+        """Return an instance of the class for the given settings"""
 
     def load(spider_name):
-        """Returns the Spider class for the given spider name. If the spider
+        """Return the Spider class for the given spider name. If the spider
         name is not found, it must raise a KeyError."""
 
     def list():
@@ -14,4 +14,9 @@ class ISpiderManager(Interface):
         project"""
 
     def find_by_request(request):
-        """Returns the list of spiders names that can handle the given request"""
+        """Return the list of spiders names that can handle the given request"""
+
+
+# ISpiderManager is deprecated, don't use it!
+# An alias is kept for backwards compatibility.
+ISpiderManager = ISpiderLoader

scrapy/settings/default_settings.py

@@ -215,7 +215,7 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue'
 
-SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager'
+SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
 SPIDER_MIDDLEWARES = {}
 

scrapy/spider.py

@@ -97,7 +97,9 @@ class ObsoleteClass(object):
     def __getattr__(self, name):
         raise AttributeError(self.message)
 
-spiders = ObsoleteClass("""
-"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings"
-""")
+spiders = ObsoleteClass(
+    '"from scrapy.spider import spiders" no longer works - use '
+    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
+    'it with your project settings"'
+)
 

scrapy/spiderloader.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from zope.interface import implementer
+
+from scrapy.interfaces import ISpiderLoader
+from scrapy.utils.misc import walk_modules
+from scrapy.utils.spider import iter_spider_classes
+
+
+@implementer(ISpiderLoader)
+class SpiderLoader(object):
+    """
+    SpiderLoader is a class which locates and loads spiders
+    in a Scrapy project.
+    """
+    def __init__(self, settings):
+        self.spider_modules = settings.getlist('SPIDER_MODULES')
+        self._spiders = {}
+        for name in self.spider_modules:
+            for module in walk_modules(name):
+                self._load_spiders(module)
+
+    def _load_spiders(self, module):
+        for spcls in iter_spider_classes(module):
+            self._spiders[spcls.name] = spcls
+
+    @classmethod
+    def from_settings(cls, settings):
+        return cls(settings)
+
+    def load(self, spider_name):
+        """
+        Return the Spider class for the given spider name. If the spider
+        name is not found, raise a KeyError.
+        """
+        try:
+            return self._spiders[spider_name]
+        except KeyError:
+            raise KeyError("Spider not found: {}".format(spider_name))
+
+    def find_by_request(self, request):
+        """
+        Return the list of spiders names that can handle the given request.
+        """
+        return [name for name, cls in self._spiders.items()
+                if cls.handles_request(request)]
+
+    def list(self):
+        """
+        Return a list with the names of all spiders available in the project.
+        """
+        return list(self._spiders.keys())

scrapy/spidermanager.py

@@ -1,43 +1,7 @@
 """
-SpiderManager is the class which locates and manages all website-specific
-spiders
+Backwards compatibility shim. Use scrapy.spiderloader instead.
 """
+from scrapy.spiderloader import SpiderLoader
+from scrapy.utils.deprecate import create_deprecated_class
 
-from zope.interface import implementer
-import six
-
-from scrapy.interfaces import ISpiderManager
-from scrapy.utils.misc import walk_modules
-from scrapy.utils.spider import iter_spider_classes
-
-
-@implementer(ISpiderManager)
-class SpiderManager(object):
-
-    def __init__(self, settings):
-        self.spider_modules = settings.getlist('SPIDER_MODULES')
-        self._spiders = {}
-        for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
-
-    def _load_spiders(self, module):
-        for spcls in iter_spider_classes(module):
-            self._spiders[spcls.name] = spcls
-
-    @classmethod
-    def from_settings(cls, settings):
-        return cls(settings)
-
-    def load(self, spider_name):
-        try:
-            return self._spiders[spider_name]
-        except KeyError:
-            raise KeyError("Spider not found: {}".format(spider_name))
-
-    def find_by_request(self, request):
-        return [name for name, cls in six.iteritems(self._spiders)
-            if cls.handles_request(request)]
-
-    def list(self):
-        return list(self._spiders.keys())
+SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)

scrapy/utils/spider.py

@@ -26,21 +26,21 @@ def iter_spider_classes(module):
            getattr(obj, 'name', None):
             yield obj
 
-def spidercls_for_request(spidermanager, request, default_spidercls=None,
+def spidercls_for_request(spiderloader, request, default_spidercls=None,
                           log_none=False, log_multiple=False):
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using
-    the spider manager) and return a Spider class if (and only if) there is
+    the spider loader) and return a Spider class if (and only if) there is
     only one Spider able to handle the Request.
 
     If multiple spiders (or no spider) are found, it will return the
     default_spidercls passed. It can optionally log if multiple or no spiders
     are found.
     """
-    snames = spidermanager.find_by_request(request)
+    snames = spiderloader.find_by_request(request)
     if len(snames) == 1:
-        return spidermanager.load(snames[0])
+        return spiderloader.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
         log.msg(format='More than one spider can handle: %(request)s - %(snames)s',

tests/py3-ignores.txt

@@ -44,13 +44,13 @@ tests/test_selector_csstranslator.py
 tests/test_selector_lxmldocument.py
 tests/test_selector.py
 tests/test_settings/__init__.py
-tests/test_spidermanager/__init__.py
-tests/test_spidermanager/test_spiders/__init__.py
-tests/test_spidermanager/test_spiders/spider0.py
-tests/test_spidermanager/test_spiders/spider1.py
-tests/test_spidermanager/test_spiders/spider2.py
-tests/test_spidermanager/test_spiders/spider3.py
-tests/test_spidermanager/test_spiders/spider4.py
+tests/test_spiderloader/__init__.py
+tests/test_spiderloader/test_spiders/__init__.py
+tests/test_spiderloader/test_spiders/spider0.py
+tests/test_spiderloader/test_spiders/spider1.py
+tests/test_spiderloader/test_spiders/spider2.py
+tests/test_spiderloader/test_spiders/spider3.py
+tests/test_spiderloader/test_spiders/spider4.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py

tests/test_crawler.py

@@ -19,7 +19,7 @@ class CrawlerTestCase(unittest.TestCase):
             spiders = self.crawler.spiders
             self.assertEqual(len(w), 1)
             self.assertIn("Crawler.spiders", str(w[0].message))
-            sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS'])
+            sm_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sm_cls)
 
             self.crawler.spiders
@@ -54,7 +54,7 @@ class CrawlerTestCase(unittest.TestCase):
 
 
 
-def SpiderManagerWithWrongInterface(object):
+class SpiderLoaderWithWrongInterface(object):
 
     def unneeded_method(self):
         pass
@@ -64,7 +64,7 @@ class CrawlerRunnerTestCase(unittest.TestCase):
 
     def test_spider_manager_verify_interface(self):
         settings = Settings({
-            'SPIDER_MANAGER_CLASS': 'tests.test_crawler.SpiderManagerWithWrongInterface'
+            'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
         })
         with self.assertRaises(DoesNotImplement):
             CrawlerRunner(settings)

tests/test_spiderloader/__init__.py

@@ -8,14 +8,15 @@ from twisted.trial import unittest
 
 # ugly hack to avoid cyclic imports of scrapy.spider when running this test
 # alone
-from scrapy.interfaces import ISpiderManager
-from scrapy.spidermanager import SpiderManager
+from scrapy.interfaces import ISpiderLoader
+from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
 from scrapy.http import Request
 
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
-class SpiderManagerTest(unittest.TestCase):
+
+class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
@@ -25,53 +26,53 @@ class SpiderManagerTest(unittest.TestCase):
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
-        self.spiderman = SpiderManager.from_settings(settings)
+        self.spiderloader = SpiderLoader.from_settings(settings)
 
     def tearDown(self):
-        del self.spiderman
+        del self.spiderloader
         del sys.modules['test_spiders_xxx']
         sys.path.remove(self.tmpdir)
 
     def test_interface(self):
-        verifyObject(ISpiderManager, self.spiderman)
+        verifyObject(ISpiderLoader, self.spiderloader)
 
     def test_list(self):
-        self.assertEqual(set(self.spiderman.list()),
+        self.assertEqual(set(self.spiderloader.list()),
             set(['spider1', 'spider2', 'spider3']))
 
     def test_load(self):
-        spider1 = self.spiderman.load("spider1")
+        spider1 = self.spiderloader.load("spider1")
         self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy1.org/test')),
             ['spider1'])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy2.org/test')),
             ['spider2'])
-        self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))),
+        self.assertEqual(set(self.spiderloader.find_by_request(Request('http://scrapy3.org/test'))),
             set(['spider1', 'spider2']))
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy999.org/test')),
             [])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com')),
             [])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com/onlythis')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com/onlythis')),
             ['spider3'])
 
     def test_load_spider_module(self):
-        module = 'tests.test_spidermanager.test_spiders.spider1'
+        module = 'tests.test_spiderloader.test_spiders.spider1'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 1
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 1
 
     def test_load_spider_module(self):
-        prefix = 'tests.test_spidermanager.test_spiders.'
+        prefix = 'tests.test_spiderloader.test_spiders.'
         module = ','.join(prefix + s for s in ('spider1', 'spider2'))
         settings = Settings({'SPIDER_MODULES': module})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 2
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 2
 
     def test_load_base_spider(self):
-        module = 'tests.test_spidermanager.test_spiders.spider0'
+        module = 'tests.test_spiderloader.test_spiders.spider0'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 0
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 0