Merge pull request #1390 from nyov/nyov/lazyload-s3-boto

[MRG+1] lazy-load s3 boto

scrapy/__init__.py

@@ -31,13 +31,6 @@ del _monkeypatches
 optional_features = set()
 # TODO: backwards compatibility, remove for Scrapy 0.20
 optional_features.add('ssl')
-try:
-    import boto
-    del boto
-except ImportError:
-    pass
-else:
-    optional_features.add('boto')
 
 from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)

scrapy/core/downloader/handlers/s3.py

@@ -1,39 +1,44 @@
 from urlparse import unquote
 
-from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from .http import HTTPDownloadHandler
 
-try:
-    from boto.s3.connection import S3Connection
-except ImportError:
-    S3Connection = object
 
-class _v19_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, method, bucket, key, headers, *args, **kwargs):
-        return headers
+def get_s3_connection():
+    try:
+        from boto.s3.connection import S3Connection
+    except ImportError:
+        return None
 
-class _v20_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, http_request, *args, **kwargs):
-        http_request.authorize(connection=self)
-        return http_request.headers
+    class _v19_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, method, bucket, key, headers, *args, **kwargs):
+            return headers
 
-try:
-    import boto.auth
-except ImportError:
-    _S3Connection = _v19_S3Connection
-else:
-    _S3Connection = _v20_S3Connection
+    class _v20_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, http_request, *args, **kwargs):
+            http_request.authorize(connection=self)
+            return http_request.headers
+
+    try:
+        import boto.auth
+    except ImportError:
+        _S3Connection = _v19_S3Connection
+    else:
+        _S3Connection = _v20_S3Connection
+
+    return _S3Connection
 
 
 class S3DownloadHandler(object):
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler):
-        if 'boto' not in optional_features:
+
+        _S3Connection = get_s3_connection()
+        if _S3Connection is None:
             raise NotConfigured("missing boto library")
 
         if not aws_access_key_id:

scrapy/pipelines/files.py

@@ -84,6 +84,11 @@ class S3FilesStore(object):
     }
 
     def __init__(self, uri):
+        try:
+            from boto.s3.connection import S3Connection
+            self.S3Connection = S3Connection
+        except ImportError:
+            raise NotConfigured("missing boto library")
         assert uri.startswith('s3://')
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
@@ -98,10 +103,9 @@ class S3FilesStore(object):
         return self._get_boto_key(path).addCallback(_onsuccess)
 
     def _get_boto_bucket(self):
-        from boto.s3.connection import S3Connection
         # disable ssl (is_secure=False) because of this python bug:
         # http://bugs.python.org/issue5103
-        c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
+        c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
         return c.get_bucket(self.bucket, validate=False)
 
     def _get_boto_key(self, path):

tests/test_downloader_handlers.py

@@ -395,7 +395,13 @@ class HttpDownloadHandlerMock(object):
         return request
 
 class S3TestCase(unittest.TestCase):
-    skip = 'boto' not in optional_features and 'missing boto library'
+    download_handler_cls = S3DownloadHandler
+    try:
+        # can't instance without settings, but ignore that
+        download_handler_cls({})
+    except NotConfigured:
+        skip = 'missing boto library'
+    except KeyError: pass
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf