lazy-loading for DownloadHandlers

scrapy/core/downloader/handlers/__init__.py

@@ -11,8 +11,10 @@ from scrapy import signals
 class DownloadHandlers(object):
 
     def __init__(self, crawler):
-        self._handlers = {}
-        self._notconfigured = {}
+        self._crawler_settings = crawler.settings
+        self._schemes = {} # stores acceptable schemes on instancing
+        self._handlers = {} # stores instanced handlers for schemes
+        self._notconfigured = {} # remembers failed handlers
         handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
         handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
         for scheme, clspath in six.iteritems(handlers):
@@ -20,25 +22,40 @@ class DownloadHandlers(object):
             # component (extension, middleware, etc).
             if clspath is None:
                 continue
-            cls = load_object(clspath)
-            try:
-                dh = cls(crawler.settings)
-            except NotConfigured as ex:
-                self._notconfigured[scheme] = str(ex)
-            else:
-                self._handlers[scheme] = dh
+            self._schemes[scheme] = clspath
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
+    def _get_handler(self, scheme):
+        """Lazy-load the downloadhandler for a scheme
+        only on the first request for that scheme.
+        """
+        if scheme in self._handlers:
+            return self._handlers[scheme]
+        if scheme in self._notconfigured:
+            return None
+        if scheme not in self._schemes:
+            self._notconfigured[scheme] = \
+                    'no handler available for that scheme'
+            return None
+
+        dhcls = load_object(self._schemes[scheme])
+        try:
+            dh = dhcls(self._crawler_settings)
+        except NotConfigured as ex:
+            self._notconfigured[scheme] = str(ex)
+            return None
+        else:
+            self._handlers[scheme] = dh
+        return self._handlers[scheme]
+
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme
-        try:
-            handler = self._handlers[scheme].download_request
-        except KeyError:
-            msg = self._notconfigured.get(scheme, \
-                    'no handler available for that scheme')
-            raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg))
-        return handler(request, spider)
+        handler = self._get_handler(scheme)
+        if not handler:
+            raise NotSupported("Unsupported URL scheme '%s': %s" %
+                    (scheme, self._notconfigured[scheme]))
+        return handler.download_request(request, spider)
 
     @defer.inlineCallbacks
     def _close(self, *_a, **_kw):

tests/test_downloader_handlers.py

@@ -52,6 +52,9 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
@@ -59,6 +62,9 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
@@ -66,8 +72,11 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': None}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertNotIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
-        self.assertNotIn('scheme', dh._notconfigured)
+        self.assertIn('scheme', dh._notconfigured)
 
 
 class FileTestCase(unittest.TestCase):