From b2fd9bdb63c45024872813587f61007c5ae06396 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 12 Jul 2015 16:38:18 +0000
Subject: [PATCH] lazy-load s3 boto

---
 scrapy/__init__.py                    |  7 ----
 scrapy/core/downloader/handlers/s3.py | 47 +++++++++++++++------------
 scrapy/pipelines/files.py             |  8 +++--
 tests/test_downloader_handlers.py     |  8 ++++-
 4 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 10ba9544f..c0477f509 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -31,13 +31,6 @@ del _monkeypatches
 optional_features = set()
 # TODO: backwards compatibility, remove for Scrapy 0.20
 optional_features.add('ssl')
-try:
-    import boto
-    del boto
-except ImportError:
-    pass
-else:
-    optional_features.add('boto')
 
 from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index f1e2c7767..f890300c4 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,39 +1,44 @@
 from urlparse import unquote
 
-from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from .http import HTTPDownloadHandler
 
-try:
-    from boto.s3.connection import S3Connection
-except ImportError:
-    S3Connection = object
 
-class _v19_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, method, bucket, key, headers, *args, **kwargs):
-        return headers
+def get_s3_connection():
+    try:
+        from boto.s3.connection import S3Connection
+    except ImportError:
+        return None
 
-class _v20_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, http_request, *args, **kwargs):
-        http_request.authorize(connection=self)
-        return http_request.headers
+    class _v19_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, method, bucket, key, headers, *args, **kwargs):
+            return headers
 
-try:
-    import boto.auth
-except ImportError:
-    _S3Connection = _v19_S3Connection
-else:
-    _S3Connection = _v20_S3Connection
+    class _v20_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, http_request, *args, **kwargs):
+            http_request.authorize(connection=self)
+            return http_request.headers
+
+    try:
+        import boto.auth
+    except ImportError:
+        _S3Connection = _v19_S3Connection
+    else:
+        _S3Connection = _v20_S3Connection
+
+    return _S3Connection
 
 
 class S3DownloadHandler(object):
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler):
-        if 'boto' not in optional_features:
+
+        _S3Connection = get_s3_connection()
+        if _S3Connection is None:
             raise NotConfigured("missing boto library")
 
         if not aws_access_key_id:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a449793c9..308d2f3c1 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -84,6 +84,11 @@ class S3FilesStore(object):
     }
 
     def __init__(self, uri):
+        try:
+            from boto.s3.connection import S3Connection
+            self.S3Connection = S3Connection
+        except ImportError:
+            raise NotConfigured("missing boto library")
         assert uri.startswith('s3://')
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
@@ -98,10 +103,9 @@ class S3FilesStore(object):
         return self._get_boto_key(path).addCallback(_onsuccess)
 
     def _get_boto_bucket(self):
-        from boto.s3.connection import S3Connection
         # disable ssl (is_secure=False) because of this python bug:
         # http://bugs.python.org/issue5103
-        c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
+        c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
         return c.get_bucket(self.bucket, validate=False)
 
     def _get_boto_key(self, path):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c814de307..131f6edb7 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -395,7 +395,13 @@ class HttpDownloadHandlerMock(object):
         return request
 
 class S3TestCase(unittest.TestCase):
-    skip = 'boto' not in optional_features and 'missing boto library'
+    download_handler_cls = S3DownloadHandler
+    try:
+        # can't instance without settings, but ignore that
+        download_handler_cls({})
+    except NotConfigured:
+        skip = 'missing boto library'
+    except KeyError: pass
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf