Default is lazy, load_object exception handling, code improvements

scrapy/core/downloader/handlers/__init__.py

@@ -24,13 +24,7 @@ class DownloadHandlers(object):
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
-        for scheme in self._schemes:
-            path = self._schemes[scheme]
-            dhcls = load_object(path)
-            lazy = getattr(dhcls, 'lazy', False)
-            if lazy:
-                continue
-            self._load_handler(scheme, dhcls)
+            self._load_handler(scheme, skip_lazy=True)
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
@@ -46,13 +40,14 @@ class DownloadHandlers(object):
             self._notconfigured[scheme] = 'no handler available for that scheme'
             return None
 
-        path = self._schemes[scheme]
-        dhcls = load_object(path)
-        self._load_handler(scheme, dhcls)
-        return self._handlers[scheme]
+        return self._load_handler(scheme)
 
-    def _load_handler(self, scheme, dhcls):
+    def _load_handler(self, scheme, skip_lazy=False):
+        path = self._schemes[scheme]
         try:
+            dhcls = load_object(path)
+            if skip_lazy and getattr(dhcls, 'lazy', True):
+                return None
             dh = dhcls(self._crawler.settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
@@ -60,11 +55,12 @@ class DownloadHandlers(object):
         except Exception as ex:
             logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
                          {"clspath": path, "scheme": scheme},
-                         exc_info=True,  extra={'crawler': self._crawler})
+                         exc_info=True, extra={'crawler': self._crawler})
             self._notconfigured[scheme] = str(ex)
             return None
         else:
             self._handlers[scheme] = dh
+            return dh
 
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme

scrapy/core/downloader/handlers/datauri.py

@@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers
 
 
 class DataURIDownloadHandler(object):
+    lazy = False
+
     def __init__(self, settings):
         super(DataURIDownloadHandler, self).__init__()
 

scrapy/core/downloader/handlers/file.py

@@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
+
 class FileDownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         pass

scrapy/core/downloader/handlers/ftp.py

@@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol):
         self.body.close() if self.filename else self.body.seek(0)
 
 _CODE_RE = re.compile("\d+")
+
+
 class FTPDownloadHandler(object):
+    lazy = False
 
     CODE_MAPPING = {
         "550": 404,

scrapy/core/downloader/handlers/http10.py

@@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode
 
 
 class HTTP10DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])

scrapy/core/downloader/handlers/http11.py

@@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class HTTP11DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)

scrapy/core/downloader/handlers/s3.py

@@ -31,7 +31,6 @@ def _get_boto_connection():
 
 
 class S3DownloadHandler(object):
-    lazy = True
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler, **kw):

tests/test_downloader_handlers.py

@@ -43,19 +43,21 @@ from tests.spiders import SingleRequestSpider
 
 
 class DummyDH(object):
+    lazy = False
 
     def __init__(self, crawler):
         pass
 
 
 class DummyLazyDH(object):
-    lazy = True
+    # Default is lazy for backwards compatibility
 
     def __init__(self, crawler):
         pass
 
 
 class OffDH(object):
+    lazy = False
 
     def __init__(self, crawler):
         raise NotConfigured