Merge pull request #3394 from hcoura/dh-lazyloading-optional

Make lazy loading Download Handlers optional

scrapy/core/downloader/handlers/__init__.py

@@ -24,6 +24,7 @@ class DownloadHandlers(object):
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
+            self._load_handler(scheme, skip_lazy=True)
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
@@ -39,9 +40,14 @@ class DownloadHandlers(object):
             self._notconfigured[scheme] = 'no handler available for that scheme'
             return None
 
+        return self._load_handler(scheme)
+
+    def _load_handler(self, scheme, skip_lazy=False):
         path = self._schemes[scheme]
         try:
             dhcls = load_object(path)
+            if skip_lazy and getattr(dhcls, 'lazy', True):
+                return None
             dh = dhcls(self._crawler.settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
@@ -49,12 +55,12 @@ class DownloadHandlers(object):
         except Exception as ex:
             logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
                          {"clspath": path, "scheme": scheme},
-                         exc_info=True,  extra={'crawler': self._crawler})
+                         exc_info=True, extra={'crawler': self._crawler})
             self._notconfigured[scheme] = str(ex)
             return None
         else:
             self._handlers[scheme] = dh
-        return self._handlers[scheme]
+            return dh
 
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme

scrapy/core/downloader/handlers/datauri.py

@@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers
 
 
 class DataURIDownloadHandler(object):
+    lazy = False
+
     def __init__(self, settings):
         super(DataURIDownloadHandler, self).__init__()
 

scrapy/core/downloader/handlers/file.py

@@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
+
 class FileDownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         pass

scrapy/core/downloader/handlers/ftp.py

@@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol):
         self.body.close() if self.filename else self.body.seek(0)
 
 _CODE_RE = re.compile("\d+")
+
+
 class FTPDownloadHandler(object):
+    lazy = False
 
     CODE_MAPPING = {
         "550": 404,

scrapy/core/downloader/handlers/http10.py

@@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode
 
 
 class HTTP10DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])

scrapy/core/downloader/handlers/http11.py

@@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class HTTP11DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)

tests/test_downloader_handlers.py

@@ -41,13 +41,23 @@ from scrapy.exceptions import NotConfigured
 from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
+
 class DummyDH(object):
+    lazy = False
+
+    def __init__(self, crawler):
+        pass
+
+
+class DummyLazyDH(object):
+    # Default is lazy for backwards compatibility
 
     def __init__(self, crawler):
         pass
 
 
 class OffDH(object):
+    lazy = False
 
     def __init__(self, crawler):
         raise NotConfigured
@@ -60,8 +70,6 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
-            dh._get_handler(scheme)
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
@@ -70,8 +78,6 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
-            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
@@ -80,11 +86,22 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertNotIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
+        for scheme in handlers:  # force load handlers
             dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
+    def test_lazy_handlers(self):
+        handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'}
+        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        self.assertNotIn('scheme', dh._handlers)
+        for scheme in handlers:  # force load lazy handler
+            dh._get_handler(scheme)
+        self.assertIn('scheme', dh._handlers)
+        self.assertNotIn('scheme', dh._notconfigured)
+
 
 class FileTestCase(unittest.TestCase):