[backport][1.1.x] Py3 S3 botocore

Originally from https://github.com/scrapy/scrapy/pull/1761

docs/topics/feed-exports.rst

@@ -99,12 +99,12 @@ The storages backends supported out of the box are:
 
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the boto_
-library is installed.
+not available. For example, the S3 backend is only available if the botocore_
+or boto_ library is installed (Scrapy supports boto_ only on Python 2).
 
 
 .. _topics-feed-uri-params:
@@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `boto`_
+ * Required external libraries: `botocore`_ or `boto`_
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -332,4 +332,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: http://aws.amazon.com/s3/
-.. _boto: http://code.google.com/p/boto/
+.. _boto: https://github.com/boto/boto
+.. _botocore: https://github.com/boto/botocore

scrapy/core/downloader/handlers/s3.py

@@ -2,14 +2,12 @@ from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.boto import is_botocore
 from .http import HTTPDownloadHandler
 
 
-def get_s3_connection():
-    try:
-        from boto.s3.connection import S3Connection
-    except ImportError:
-        return None
+def _get_boto_connection():
+    from boto.s3.connection import S3Connection
 
     class _v19_S3Connection(S3Connection):
         """A dummy S3Connection wrapper that doesn't do any synchronous download"""
@@ -37,10 +35,6 @@ class S3DownloadHandler(object):
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler, **kw):
 
-        _S3Connection = get_s3_connection()
-        if _S3Connection is None:
-            raise NotConfigured("missing boto library")
-
         if not aws_access_key_id:
             aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
@@ -49,14 +43,30 @@ class S3DownloadHandler(object):
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
         # unless 'anon' was set explicitly (True/False).
-        anon = kw.get('anon', None)
+        anon = kw.get('anon')
         if anon is None and not aws_access_key_id and not aws_secret_access_key:
             kw['anon'] = True
+        self.anon = kw.get('anon')
+
+        self._signer = None
+        if is_botocore():
+            import botocore.auth
+            import botocore.credentials
+            kw.pop('anon', None)
+            if kw:
+                raise TypeError('Unexpected keyword arguments: %s' % kw)
+            if not self.anon:
+                SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+                self._signer = SignerCls(botocore.credentials.Credentials(
+                    aws_access_key_id, aws_secret_access_key))
+        else:
+            _S3Connection = _get_boto_connection()
+            try:
+                self.conn = _S3Connection(
+                    aws_access_key_id, aws_secret_access_key, **kw)
+            except Exception as ex:
+                raise NotConfigured(str(ex))
 
-        try:
-            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)
-        except Exception as ex:
-            raise NotConfigured(str(ex))
         self._download_http = httpdownloadhandler(settings).download_request
 
     def download_request(self, request, spider):
@@ -65,12 +75,25 @@ class S3DownloadHandler(object):
         bucket = p.hostname
         path = p.path + '?' + p.query if p.query else p.path
         url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
-        signed_headers = self.conn.make_request(
+        if self.anon:
+            request = request.replace(url=url)
+        elif self._signer is not None:
+            import botocore.awsrequest
+            awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
-                bucket=bucket,
-                key=unquote(p.path),
-                query_args=unquote(p.query),
-                headers=request.headers,
+                url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
+                headers=request.headers.to_unicode_dict(),
                 data=request.body)
-        httpreq = request.replace(url=url, headers=signed_headers)
-        return self._download_http(httpreq, spider)
+            self._signer.add_auth(awsrequest)
+            request = request.replace(
+                url=url, headers=awsrequest.headers.items())
+        else:
+            signed_headers = self.conn.make_request(
+                    method=request.method,
+                    bucket=bucket,
+                    key=unquote(p.path),
+                    query_args=unquote(p.query),
+                    headers=request.headers,
+                    data=request.body)
+            request = request.replace(url=url, headers=signed_headers)
+        return self._download_http(request, spider)

scrapy/extensions/feedexport.py

@@ -23,6 +23,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import without_none_values
+from scrapy.utils.boto import is_botocore
 
 logger = logging.getLogger(__name__)
 
@@ -87,24 +88,33 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):
         from scrapy.conf import settings
-        try:
-            import boto
-        except ImportError:
-            raise NotConfigured
-        self.connect_s3 = boto.connect_s3
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
         self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
-        self.keyname = u.path
+        self.is_botocore = is_botocore()
+        self.keyname = u.path[1:]  # remove first "/"
+        if self.is_botocore:
+            import botocore.session
+            session = botocore.session.get_session()
+            self.s3_client = session.create_client(
+                's3', aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key)
+        else:
+            import boto
+            self.connect_s3 = boto.connect_s3
 
     def _store_in_thread(self, file):
         file.seek(0)
-        conn = self.connect_s3(self.access_key, self.secret_key)
-        bucket = conn.get_bucket(self.bucketname, validate=False)
-        key = bucket.new_key(self.keyname)
-        key.set_contents_from_file(file)
-        key.close()
+        if self.is_botocore:
+            self.s3_client.put_object(
+                Bucket=self.bucketname, Key=self.keyname, Body=file)
+        else:
+            conn = self.connect_s3(self.access_key, self.secret_key)
+            bucket = conn.get_bucket(self.bucketname, validate=False)
+            key = bucket.new_key(self.keyname)
+            key.set_contents_from_file(file)
+            key.close()
 
 
 class FTPFeedStorage(BlockingFeedStorage):

scrapy/http/headers.py

@@ -1,6 +1,7 @@
 import six
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.python import to_unicode
 
 
 class Headers(CaselessDict):
@@ -78,6 +79,15 @@ class Headers(CaselessDict):
     def to_string(self):
         return headers_dict_to_raw(self)
 
+    def to_unicode_dict(self):
+        """ Return headers as a CaselessDict with unicode keys
+        and unicode values. Multiple values are joined with ','.
+        """
+        return CaselessDict(
+            (to_unicode(key, encoding=self.encoding),
+             to_unicode(b','.join(value), encoding=self.encoding))
+            for key, value in self.items())
+
     def __copy__(self):
         return self.__class__(self)
     copy = __copy__

scrapy/pipelines/files.py

@@ -28,6 +28,8 @@ from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
+from scrapy.utils.boto import is_botocore
+from scrapy.utils.datatypes import CaselessDict
 
 logger = logging.getLogger(__name__)
 
@@ -86,20 +88,30 @@ class S3FilesStore(object):
     }
 
     def __init__(self, uri):
-        try:
+        self.is_botocore = is_botocore()
+        if self.is_botocore:
+            import botocore.session
+            session = botocore.session.get_session()
+            self.s3_client = session.create_client(
+                's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY)
+        else:
             from boto.s3.connection import S3Connection
             self.S3Connection = S3Connection
-        except ImportError:
-            raise NotConfigured("missing boto library")
         assert uri.startswith('s3://')
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
     def stat_file(self, path, info):
         def _onsuccess(boto_key):
-            checksum = boto_key.etag.strip('"')
-            last_modified = boto_key.last_modified
-            modified_tuple = parsedate_tz(last_modified)
-            modified_stamp = int(mktime_tz(modified_tuple))
+            if self.is_botocore:
+                checksum = boto_key['ETag'].strip('"')
+                last_modified = boto_key['LastModified']
+                modified_stamp = time.mktime(last_modified.timetuple())
+            else:
+                checksum = boto_key.etag.strip('"')
+                last_modified = boto_key.last_modified
+                modified_tuple = parsedate_tz(last_modified)
+                modified_stamp = int(mktime_tz(modified_tuple))
             return {'checksum': checksum, 'last_modified': modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
@@ -111,24 +123,73 @@ class S3FilesStore(object):
         return c.get_bucket(self.bucket, validate=False)
 
     def _get_boto_key(self, path):
-        b = self._get_boto_bucket()
         key_name = '%s%s' % (self.prefix, path)
-        return threads.deferToThread(b.get_key, key_name)
+        if self.is_botocore:
+            return threads.deferToThread(
+                self.s3_client.head_object,
+                Bucket=self.bucket,
+                Key=key_name)
+        else:
+            b = self._get_boto_bucket()
+            return threads.deferToThread(b.get_key, key_name)
 
     def persist_file(self, path, buf, info, meta=None, headers=None):
         """Upload file to S3 storage"""
-        b = self._get_boto_bucket()
         key_name = '%s%s' % (self.prefix, path)
-        k = b.new_key(key_name)
-        if meta:
-            for metakey, metavalue in six.iteritems(meta):
-                k.set_metadata(metakey, str(metavalue))
-        h = self.HEADERS.copy()
-        if headers:
-            h.update(headers)
         buf.seek(0)
-        return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
-                                     headers=h, policy=self.POLICY)
+        if self.is_botocore:
+            extra = self._headers_to_botocore_kwargs(self.HEADERS)
+            if headers:
+                extra.update(self._headers_to_botocore_kwargs(headers))
+            return threads.deferToThread(
+                self.s3_client.put_object,
+                Bucket=self.bucket,
+                Key=key_name,
+                Body=buf,
+                Metadata={k: str(v) for k, v in six.iteritems(meta)},
+                ACL=self.POLICY,
+                **extra)
+        else:
+            b = self._get_boto_bucket()
+            k = b.new_key(key_name)
+            if meta:
+                for metakey, metavalue in six.iteritems(meta):
+                    k.set_metadata(metakey, str(metavalue))
+            h = self.HEADERS.copy()
+            if headers:
+                h.update(headers)
+            return threads.deferToThread(
+                k.set_contents_from_string, buf.getvalue(),
+                headers=h, policy=self.POLICY)
+
+    def _headers_to_botocore_kwargs(self, headers):
+        """ Convert headers to botocore keyword agruments.
+        """
+        # This is required while we need to support both boto and botocore.
+        mapping = CaselessDict({
+            'Content-Type': 'ContentType',
+            'Cache-Control': 'CacheControl',
+            'Content-Disposition': 'ContentDisposition',
+            'Content-Encoding': 'ContentEncoding',
+            'Content-Language': 'ContentLanguage',
+            'Content-Length': 'ContentLength',
+            'Content-MD5': 'ContentMD5',
+            'Expires': 'Expires',
+            'X-Amz-Grant-Full-Control': 'GrantFullControl',
+            'X-Amz-Grant-Read': 'GrantRead',
+            'X-Amz-Grant-Read-ACP': 'GrantReadACP',
+            'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+            })
+        extra = {}
+        for key, value in six.iteritems(headers):
+            try:
+                kwarg = mapping[key]
+            except KeyError:
+                raise TypeError(
+                    'Header "%s" is not supported by botocore' % key)
+            else:
+                extra[kwarg] = value
+        return extra
 
 
 class FilesPipeline(MediaPipeline):

scrapy/utils/boto.py

@@ -0,0 +1,21 @@
+"""Boto/botocore helpers"""
+
+from __future__ import absolute_import
+import six
+
+from scrapy.exceptions import NotConfigured
+
+
+def is_botocore():
+    try:
+        import botocore
+        return True
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+                return False
+            except ImportError:
+                raise NotConfigured('missing botocore or boto library')
+        else:
+            raise NotConfigured('missing botocore library')

scrapy/utils/test.py

@@ -2,24 +2,49 @@
 This module contains some assorted functions used in tests
 """
 
+from __future__ import absolute_import
 import os
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
 
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.boto import is_botocore
+
 
 def assert_aws_environ():
     """Asserts the current environment is suitable for running AWS testsi.
     Raises SkipTest with the reason if it's not.
     """
-    try:
-        import boto
-    except ImportError as e:
-        raise SkipTest(str(e))
-
+    skip_if_no_boto()
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+def skip_if_no_boto():
+    try:
+        is_botocore()
+    except NotConfigured as e:
+        raise SkipTest(e.message)
+
+def get_s3_content_and_delete(bucket, path, with_key=False):
+    """ Get content from s3 key, and delete key afterwards.
+    """
+    if is_botocore():
+        import botocore.session
+        session = botocore.session.get_session()
+        client = session.create_client('s3')
+        key = client.get_object(Bucket=bucket, Key=path)
+        content = key['Body'].read()
+        client.delete_object(Bucket=bucket, Key=path)
+    else:
+        import boto
+        # assuming boto=2.2.2
+        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
+        key = bucket.get_key(path)
+        content = key.get_contents_as_string()
+        bucket.delete_key(path)
+    return (content, key) if with_key else content
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level

tests/py3-ignores.txt

@@ -7,7 +7,6 @@ scrapy/xlib/tx/endpoints.py
 scrapy/xlib/tx/client.py
 scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
-scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/ftp.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py

tests/requirements-py3.txt

@@ -4,7 +4,7 @@ pytest-cov
 testfixtures
 jmespath
 leveldb
-boto
+botocore
 # optional for shell wrapper tests
 bpython
 ipython

tests/test_downloader_handlers.py

@@ -1,5 +1,10 @@
 import os
 import six
+import contextlib
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -23,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
@@ -433,14 +438,9 @@ class HttpDownloadHandlerMock(object):
 
 
 class S3AnonTestCase(unittest.TestCase):
-    try:
-        import boto
-    except ImportError:
-        skip = 'missing boto library'
-    if six.PY3:
-        skip = 'S3 not supported on Py3'
 
     def setUp(self):
+        skip_if_no_boto()
         self.s3reqh = S3DownloadHandler(Settings(),
                 httpdownloadhandler=HttpDownloadHandlerMock,
                 #anon=True, # is implicit
@@ -451,18 +451,14 @@ class S3AnonTestCase(unittest.TestCase):
     def test_anon_request(self):
         req = Request('s3://aws-publicdatasets/')
         httpreq = self.download_request(req, self.spider)
-        self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
-        self.assertEqual(self.s3reqh.conn.anon, True)
+        self.assertEqual(hasattr(self.s3reqh, 'anon'), True)
+        self.assertEqual(self.s3reqh.anon, True)
+        self.assertEqual(
+            httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
 class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
-    try:
-        import boto
-    except ImportError:
-        skip = 'missing boto library'
-    if six.PY3:
-        skip = 'S3 not supported on Py3'
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
@@ -472,69 +468,108 @@ class S3TestCase(unittest.TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
+        skip_if_no_boto()
         s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
                 self.AWS_SECRET_ACCESS_KEY,
                 httpdownloadhandler=HttpDownloadHandlerMock)
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')
 
+    @contextlib.contextmanager
+    def _mocked_date(self, date):
+        try:
+            import botocore.auth
+        except ImportError:
+            yield
+        else:
+            # We need to mock botocore.auth.formatdate, because otherwise
+            # botocore overrides Date header with current date and time
+            # and Authorization header is different each time
+            with mock.patch('botocore.auth.formatdate') as mock_formatdate:
+                mock_formatdate.return_value = date
+                yield
+
+    def test_extra_kw(self):
+        try:
+            S3DownloadHandler(Settings(), extra_kw=True)
+        except Exception as e:
+            self.assertIsInstance(e, (TypeError, NotConfigured))
+        else:
+            assert False
+
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
-        req = Request('s3://johnsmith/photos/puppy.jpg',
-                headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date ='Tue, 27 Mar 2007 19:36:42 +0000'
+        req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 21:15:45 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
             'Content-Type': 'image/jpeg',
-            'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
+            'Date': date,
             'Content-Length': '94328',
             })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
                 method='GET', headers={
                     'User-Agent': 'Mozilla/5.0',
-                    'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+                    'Date': date,
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
 
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
-        req = Request('s3://johnsmith/?acl', \
-                method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date = 'Tue, 27 Mar 2007 19:44:46 +0000'
+        req = Request('s3://johnsmith/?acl',
+            method='GET', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
+        try: import botocore
+        except ImportError: pass
+        else:
+            raise unittest.SkipTest(
+                'botocore does not support overriding date with x-amz-date')
         # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
+        date = 'Tue, 27 Mar 2007 21:20:27 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', \
                 method='DELETE', headers={
-                    'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
+                    'Date': date,
                     'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
                     })
-        httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        # botocore does not override Date with x-amz-date
+        self.assertEqual(httpreq.headers['Authorization'],
+                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
+        date = 'Tue, 27 Mar 2007 21:06:08 +0000'
         req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
                 method='PUT', headers={
                     'User-Agent': 'curl/7.15.5',
                     'Host': 'static.johnsmith.net:8080',
-                    'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
+                    'Date': date,
                     'x-amz-acl': 'public-read',
                     'content-type': 'application/x-download',
                     'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
@@ -545,23 +580,25 @@ class S3TestCase(unittest.TestCase):
                     'Content-Encoding': 'gzip',
                     'Content-Length': '5913339',
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
     def test_request_signing7(self):
         # ensure that spaces are quoted properly before signing
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
             ("s3://johnsmith/photos/my puppy.jpg"
              "?response-content-disposition=my puppy.jpg"),
             method='GET',
-            headers={
-                'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
-            })
-        httpreq = self.download_request(req, self.spider)
+            headers={'Date': date},
+            )
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(
             httpreq.headers['Authorization'],
-            'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
+            b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
 
 
 class FTPTestCase(unittest.TestCase):

tests/test_feedexport.py

@@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
     S3FeedStorage, StdoutFeedStorage
 )
-from scrapy.utils.test import assert_aws_environ
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.python import to_native_str
 
 
@@ -92,18 +92,18 @@ class S3FeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         assert_aws_environ()
-        uri = os.environ.get('FEEDTEST_S3_URI')
+        uri = os.environ.get('S3_TEST_FILE_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
-        from boto import connect_s3
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
-        file.write("content")
+        expected_content = b"content: \xe2\x98\x83"
+        file.write(expected_content)
         yield storage.store(file)
         u = urlparse(uri)
-        key = connect_s3().get_bucket(u.hostname, validate=False).get_key(u.path)
-        self.assertEqual(key.get_contents_as_string(), "content")
+        content = get_s3_content_and_delete(u.hostname, u.path[1:])
+        self.assertEqual(content, expected_content)
 
 
 class StdoutFeedStorageTest(unittest.TestCase):

tests/test_pipeline_files.py

@@ -4,15 +4,19 @@ import hashlib
 import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from six.moves.urllib.parse import urlparse
+from six import BytesIO
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.boto import is_botocore
 
 from tests import mock
 
@@ -179,6 +183,41 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
             self.assertEqual(item['stored_file'], [results[0][1]])
 
 
+class TestS3FilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_aws_environ()
+        uri = os.environ.get('S3_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No S3 URI available for testing")
+        data = b"TestS3FilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = ''
+        store = S3FilesStore(uri)
+        yield store.persist_file(
+            path, buf, info=None, meta=meta,
+            headers={'Content-Type': 'image/png'})
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
+        u = urlparse(uri)
+        content, key = get_s3_content_and_delete(
+            u.hostname, u.path[1:], with_key=True)
+        self.assertEqual(content, data)
+        if is_botocore():
+            self.assertEqual(key['Metadata'], {'foo': 'bar'})
+            self.assertEqual(
+                key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
+            self.assertEqual(key['ContentType'], 'image/png')
+        else:
+            self.assertEqual(key.metadata, {'foo': 'bar'})
+            self.assertEqual(
+                key.cache_control, S3FilesStore.HEADERS['Cache-Control'])
+            self.assertEqual(key.content_type, 'image/png')
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()

tox.ini

@@ -10,10 +10,14 @@ envlist = py27
 deps =
     -rrequirements.txt
     # Extras
-    boto
+    botocore
     Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt
+passenv =
+    S3_TEST_FILE_URI
+    AWS_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}