From 25e481fd14b2b97895ee04f4a0a0b5c080d5b2f1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 5 Feb 2016 12:52:59 +0300 Subject: [PATCH] [backport][1.1.x] Py3 S3 botocore Originally from https://github.com/scrapy/scrapy/pull/1761 --- docs/topics/feed-exports.rst | 11 +-- scrapy/core/downloader/handlers/s3.py | 65 +++++++++----- scrapy/extensions/feedexport.py | 32 ++++--- scrapy/http/headers.py | 10 +++ scrapy/pipelines/files.py | 99 +++++++++++++++++---- scrapy/utils/boto.py | 21 +++++ scrapy/utils/test.py | 35 ++++++-- tests/py3-ignores.txt | 1 - tests/requirements-py3.txt | 2 +- tests/test_downloader_handlers.py | 119 +++++++++++++++++--------- tests/test_feedexport.py | 12 +-- tests/test_pipeline_files.py | 41 ++++++++- tox.ini | 6 +- 13 files changed, 342 insertions(+), 112 deletions(-) create mode 100644 scrapy/utils/boto.py diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 03c6fb3fb..e5037129c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -99,12 +99,12 @@ The storages backends supported out of the box are: * :ref:`topics-feed-storage-fs` * :ref:`topics-feed-storage-ftp` - * :ref:`topics-feed-storage-s3` (requires boto_) + * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_) * :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the boto_ -library is installed. +not available. For example, the S3 backend is only available if the botocore_ +or boto_ library is installed (Scrapy supports boto_ only on Python 2). .. _topics-feed-uri-params: @@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `boto`_ + * Required external libraries: `botocore`_ or `boto`_ The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -332,4 +332,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: http://aws.amazon.com/s3/ -.. _boto: http://code.google.com/p/boto/ +.. _boto: https://github.com/boto/boto +.. _botocore: https://github.com/boto/botocore diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 38cfd1e10..d8bbdd326 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,14 +2,12 @@ from six.moves.urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.boto import is_botocore from .http import HTTPDownloadHandler -def get_s3_connection(): - try: - from boto.s3.connection import S3Connection - except ImportError: - return None +def _get_boto_connection(): + from boto.s3.connection import S3Connection class _v19_S3Connection(S3Connection): """A dummy S3Connection wrapper that doesn't do any synchronous download""" @@ -37,10 +35,6 @@ class S3DownloadHandler(object): def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): - _S3Connection = get_s3_connection() - if _S3Connection is None: - raise NotConfigured("missing boto library") - if not aws_access_key_id: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: @@ -49,14 +43,30 @@ class S3DownloadHandler(object): # If no credentials could be found anywhere, # consider this an anonymous connection request by default; # unless 'anon' was set explicitly (True/False). - anon = kw.get('anon', None) + anon = kw.get('anon') if anon is None and not aws_access_key_id and not aws_secret_access_key: kw['anon'] = True + self.anon = kw.get('anon') + + self._signer = None + if is_botocore(): + import botocore.auth + import botocore.credentials + kw.pop('anon', None) + if kw: + raise TypeError('Unexpected keyword arguments: %s' % kw) + if not self.anon: + SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] + self._signer = SignerCls(botocore.credentials.Credentials( + aws_access_key_id, aws_secret_access_key)) + else: + _S3Connection = _get_boto_connection() + try: + self.conn = _S3Connection( + aws_access_key_id, aws_secret_access_key, **kw) + except Exception as ex: + raise NotConfigured(str(ex)) - try: - self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw) - except Exception as ex: - raise NotConfigured(str(ex)) self._download_http = httpdownloadhandler(settings).download_request def download_request(self, request, spider): @@ -65,12 +75,25 @@ class S3DownloadHandler(object): bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) - signed_headers = self.conn.make_request( + if self.anon: + request = request.replace(url=url) + elif self._signer is not None: + import botocore.awsrequest + awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, + url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), + headers=request.headers.to_unicode_dict(), data=request.body) - httpreq = request.replace(url=url, headers=signed_headers) - return self._download_http(httpreq, spider) + self._signer.add_auth(awsrequest) + request = request.replace( + url=url, headers=awsrequest.headers.items()) + else: + signed_headers = self.conn.make_request( + method=request.method, + bucket=bucket, + key=unquote(p.path), + query_args=unquote(p.query), + headers=request.headers, + data=request.body) + request = request.replace(url=url, headers=signed_headers) + return self._download_http(request, spider) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index daea551cb..6c54bd38e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -23,6 +23,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import without_none_values +from scrapy.utils.boto import is_botocore logger = logging.getLogger(__name__) @@ -87,24 +88,33 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri): from scrapy.conf import settings - try: - import boto - except ImportError: - raise NotConfigured - self.connect_s3 = boto.connect_s3 u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or settings['AWS_ACCESS_KEY_ID'] self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY'] - self.keyname = u.path + self.is_botocore = is_botocore() + self.keyname = u.path[1:] # remove first "/" + if self.is_botocore: + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key) + else: + import boto + self.connect_s3 = boto.connect_s3 def _store_in_thread(self, file): file.seek(0) - conn = self.connect_s3(self.access_key, self.secret_key) - bucket = conn.get_bucket(self.bucketname, validate=False) - key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) - key.close() + if self.is_botocore: + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file) + else: + conn = self.connect_s3(self.access_key, self.secret_key) + bucket = conn.get_bucket(self.bucketname, validate=False) + key = bucket.new_key(self.keyname) + key.set_contents_from_file(file) + key.close() class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 13f0f0383..62507eb19 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ import six from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.python import to_unicode class Headers(CaselessDict): @@ -78,6 +79,15 @@ class Headers(CaselessDict): def to_string(self): return headers_dict_to_raw(self) + def to_unicode_dict(self): + """ Return headers as a CaselessDict with unicode keys + and unicode values. Multiple values are joined with ','. + """ + return CaselessDict( + (to_unicode(key, encoding=self.encoding), + to_unicode(b','.join(value), encoding=self.encoding)) + for key, value in self.items()) + def __copy__(self): return self.__class__(self) copy = __copy__ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index e4011d31d..45ceddcbb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -28,6 +28,8 @@ from scrapy.utils.misc import md5sum from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str +from scrapy.utils.boto import is_botocore +from scrapy.utils.datatypes import CaselessDict logger = logging.getLogger(__name__) @@ -86,20 +88,30 @@ class S3FilesStore(object): } def __init__(self, uri): - try: + self.is_botocore = is_botocore() + if self.is_botocore: + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY) + else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection - except ImportError: - raise NotConfigured("missing boto library") assert uri.startswith('s3://') self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): def _onsuccess(boto_key): - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = parsedate_tz(last_modified) - modified_stamp = int(mktime_tz(modified_tuple)) + if self.is_botocore: + checksum = boto_key['ETag'].strip('"') + last_modified = boto_key['LastModified'] + modified_stamp = time.mktime(last_modified.timetuple()) + else: + checksum = boto_key.etag.strip('"') + last_modified = boto_key.last_modified + modified_tuple = parsedate_tz(last_modified) + modified_stamp = int(mktime_tz(modified_tuple)) return {'checksum': checksum, 'last_modified': modified_stamp} return self._get_boto_key(path).addCallback(_onsuccess) @@ -111,24 +123,73 @@ class S3FilesStore(object): return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, path): - b = self._get_boto_bucket() key_name = '%s%s' % (self.prefix, path) - return threads.deferToThread(b.get_key, key_name) + if self.is_botocore: + return threads.deferToThread( + self.s3_client.head_object, + Bucket=self.bucket, + Key=key_name) + else: + b = self._get_boto_bucket() + return threads.deferToThread(b.get_key, key_name) def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - b = self._get_boto_bucket() key_name = '%s%s' % (self.prefix, path) - k = b.new_key(key_name) - if meta: - for metakey, metavalue in six.iteritems(meta): - k.set_metadata(metakey, str(metavalue)) - h = self.HEADERS.copy() - if headers: - h.update(headers) buf.seek(0) - return threads.deferToThread(k.set_contents_from_string, buf.getvalue(), - headers=h, policy=self.POLICY) + if self.is_botocore: + extra = self._headers_to_botocore_kwargs(self.HEADERS) + if headers: + extra.update(self._headers_to_botocore_kwargs(headers)) + return threads.deferToThread( + self.s3_client.put_object, + Bucket=self.bucket, + Key=key_name, + Body=buf, + Metadata={k: str(v) for k, v in six.iteritems(meta)}, + ACL=self.POLICY, + **extra) + else: + b = self._get_boto_bucket() + k = b.new_key(key_name) + if meta: + for metakey, metavalue in six.iteritems(meta): + k.set_metadata(metakey, str(metavalue)) + h = self.HEADERS.copy() + if headers: + h.update(headers) + return threads.deferToThread( + k.set_contents_from_string, buf.getvalue(), + headers=h, policy=self.POLICY) + + def _headers_to_botocore_kwargs(self, headers): + """ Convert headers to botocore keyword agruments. + """ + # This is required while we need to support both boto and botocore. + mapping = CaselessDict({ + 'Content-Type': 'ContentType', + 'Cache-Control': 'CacheControl', + 'Content-Disposition': 'ContentDisposition', + 'Content-Encoding': 'ContentEncoding', + 'Content-Language': 'ContentLanguage', + 'Content-Length': 'ContentLength', + 'Content-MD5': 'ContentMD5', + 'Expires': 'Expires', + 'X-Amz-Grant-Full-Control': 'GrantFullControl', + 'X-Amz-Grant-Read': 'GrantRead', + 'X-Amz-Grant-Read-ACP': 'GrantReadACP', + 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', + }) + extra = {} + for key, value in six.iteritems(headers): + try: + kwarg = mapping[key] + except KeyError: + raise TypeError( + 'Header "%s" is not supported by botocore' % key) + else: + extra[kwarg] = value + return extra class FilesPipeline(MediaPipeline): diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py new file mode 100644 index 000000000..421ab2f7e --- /dev/null +++ b/scrapy/utils/boto.py @@ -0,0 +1,21 @@ +"""Boto/botocore helpers""" + +from __future__ import absolute_import +import six + +from scrapy.exceptions import NotConfigured + + +def is_botocore(): + try: + import botocore + return True + except ImportError: + if six.PY2: + try: + import boto + return False + except ImportError: + raise NotConfigured('missing botocore or boto library') + else: + raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 51edfd353..bf66a8cbe 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,24 +2,49 @@ This module contains some assorted functions used in tests """ +from __future__ import absolute_import import os from importlib import import_module from twisted.trial.unittest import SkipTest +from scrapy.exceptions import NotConfigured +from scrapy.utils.boto import is_botocore + def assert_aws_environ(): """Asserts the current environment is suitable for running AWS testsi. Raises SkipTest with the reason if it's not. """ - try: - import boto - except ImportError as e: - raise SkipTest(str(e)) - + skip_if_no_boto() if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") +def skip_if_no_boto(): + try: + is_botocore() + except NotConfigured as e: + raise SkipTest(e.message) + +def get_s3_content_and_delete(bucket, path, with_key=False): + """ Get content from s3 key, and delete key afterwards. + """ + if is_botocore(): + import botocore.session + session = botocore.session.get_session() + client = session.create_client('s3') + key = client.get_object(Bucket=bucket, Key=path) + content = key['Body'].read() + client.delete_object(Bucket=bucket, Key=path) + else: + import boto + # assuming boto=2.2.2 + bucket = boto.connect_s3().get_bucket(bucket, validate=False) + key = bucket.get_key(path) + content = key.get_contents_as_string() + bucket.delete_key(path) + return (content, key) if with_key else content + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 6385ad3b1..ec2947003 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -7,7 +7,6 @@ scrapy/xlib/tx/endpoints.py scrapy/xlib/tx/client.py scrapy/xlib/tx/_newclient.py scrapy/xlib/tx/__init__.py -scrapy/core/downloader/handlers/s3.py scrapy/core/downloader/handlers/ftp.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 73e73e651..2a89763a5 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,7 +4,7 @@ pytest-cov testfixtures jmespath leveldb -boto +botocore # optional for shell wrapper tests bpython ipython diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 56608bfc6..1885a53a0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,10 @@ import os import six +import contextlib +try: + from unittest import mock +except ImportError: + import mock from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -23,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured @@ -433,14 +438,9 @@ class HttpDownloadHandlerMock(object): class S3AnonTestCase(unittest.TestCase): - try: - import boto - except ImportError: - skip = 'missing boto library' - if six.PY3: - skip = 'S3 not supported on Py3' def setUp(self): + skip_if_no_boto() self.s3reqh = S3DownloadHandler(Settings(), httpdownloadhandler=HttpDownloadHandlerMock, #anon=True, # is implicit @@ -451,18 +451,14 @@ class S3AnonTestCase(unittest.TestCase): def test_anon_request(self): req = Request('s3://aws-publicdatasets/') httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) - self.assertEqual(self.s3reqh.conn.anon, True) + self.assertEqual(hasattr(self.s3reqh, 'anon'), True) + self.assertEqual(self.s3reqh.anon, True) + self.assertEqual( + httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/') class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler - try: - import boto - except ImportError: - skip = 'missing boto library' - if six.PY3: - skip = 'S3 not supported on Py3' # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf @@ -472,69 +468,108 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' def setUp(self): + skip_if_no_boto() s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request self.spider = Spider('foo') + @contextlib.contextmanager + def _mocked_date(self, date): + try: + import botocore.auth + except ImportError: + yield + else: + # We need to mock botocore.auth.formatdate, because otherwise + # botocore overrides Date header with current date and time + # and Authorization header is different each time + with mock.patch('botocore.auth.formatdate') as mock_formatdate: + mock_formatdate.return_value = date + yield + + def test_extra_kw(self): + try: + S3DownloadHandler(Settings(), extra_kw=True) + except Exception as e: + self.assertIsInstance(e, (TypeError, NotConfigured)) + else: + assert False + def test_request_signing1(self): # gets an object from the johnsmith bucket. - req = Request('s3://johnsmith/photos/puppy.jpg', - headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'}) - httpreq = self.download_request(req, self.spider) + date ='Tue, 27 Mar 2007 19:36:42 +0000' + req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') def test_request_signing2(self): # puts an object into the johnsmith bucket. + date = 'Tue, 27 Mar 2007 21:15:45 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ 'Content-Type': 'image/jpeg', - 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Date': date, 'Content-Length': '94328', }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') def test_request_signing3(self): # lists the content of the johnsmith bucket. + date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ method='GET', headers={ 'User-Agent': 'Mozilla/5.0', - 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', + 'Date': date, }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. - req = Request('s3://johnsmith/?acl', \ - method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'}) - httpreq = self.download_request(req, self.spider) + date = 'Tue, 27 Mar 2007 19:44:46 +0000' + req = Request('s3://johnsmith/?acl', + method='GET', headers={'Date': date}) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): + try: import botocore + except ImportError: pass + else: + raise unittest.SkipTest( + 'botocore does not support overriding date with x-amz-date') # deletes an object from the 'johnsmith' bucket using the # path-style and Date alternative. + date = 'Tue, 27 Mar 2007 21:20:27 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', \ method='DELETE', headers={ - 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000', + 'Date': date, 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', }) - httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) + # botocore does not override Date with x-amz-date + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. + date = 'Tue, 27 Mar 2007 21:06:08 +0000' req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ method='PUT', headers={ 'User-Agent': 'curl/7.15.5', 'Host': 'static.johnsmith.net:8080', - 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000', + 'Date': date, 'x-amz-acl': 'public-read', 'content-type': 'application/x-download', 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', @@ -545,23 +580,25 @@ class S3TestCase(unittest.TestCase): 'Content-Encoding': 'gzip', 'Content-Length': '5913339', }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ - 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') def test_request_signing7(self): # ensure that spaces are quoted properly before signing + date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request( ("s3://johnsmith/photos/my puppy.jpg" "?response-content-disposition=my puppy.jpg"), method='GET', - headers={ - 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', - }) - httpreq = self.download_request(req, self.spider) + headers={'Date': date}, + ) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual( httpreq.headers['Authorization'], - 'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=') + b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=') class FTPTestCase(unittest.TestCase): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 176fd93e3..c76d26b57 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage ) -from scrapy.utils.test import assert_aws_environ +from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete from scrapy.utils.python import to_native_str @@ -92,18 +92,18 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): assert_aws_environ() - uri = os.environ.get('FEEDTEST_S3_URI') + uri = os.environ.get('S3_TEST_FILE_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") - from boto import connect_s3 storage = S3FeedStorage(uri) verifyObject(IFeedStorage, storage) file = storage.open(scrapy.Spider("default")) - file.write("content") + expected_content = b"content: \xe2\x98\x83" + file.write(expected_content) yield storage.store(file) u = urlparse(uri) - key = connect_s3().get_bucket(u.hostname, validate=False).get_key(u.path) - self.assertEqual(key.get_contents_as_string(), "content") + content = get_s3_content_and_delete(u.hostname, u.path[1:]) + self.assertEqual(content, expected_content) class StdoutFeedStorageTest(unittest.TestCase): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c9977f5ca..77e75d5ac 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -4,15 +4,19 @@ import hashlib import warnings from tempfile import mkdtemp from shutil import rmtree +from six.moves.urllib.parse import urlparse +from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes +from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete +from scrapy.utils.boto import is_botocore from tests import mock @@ -179,6 +183,41 @@ class FilesPipelineTestCaseFields(unittest.TestCase): self.assertEqual(item['stored_file'], [results[0][1]]) +class TestS3FilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_aws_environ() + uri = os.environ.get('S3_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No S3 URI available for testing") + data = b"TestS3FilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = '' + store = S3FilesStore(uri) + yield store.persist_file( + path, buf, info=None, meta=meta, + headers={'Content-Type': 'image/png'}) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8') + u = urlparse(uri) + content, key = get_s3_content_and_delete( + u.hostname, u.path[1:], with_key=True) + self.assertEqual(content, data) + if is_botocore(): + self.assertEqual(key['Metadata'], {'foo': 'bar'}) + self.assertEqual( + key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) + self.assertEqual(key['ContentType'], 'image/png') + else: + self.assertEqual(key.metadata, {'foo': 'bar'}) + self.assertEqual( + key.cache_control, S3FilesStore.HEADERS['Cache-Control']) + self.assertEqual(key.content_type, 'image/png') + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tox.ini b/tox.ini index b8d45d5b9..2a8067618 100644 --- a/tox.ini +++ b/tox.ini @@ -10,10 +10,14 @@ envlist = py27 deps = -rrequirements.txt # Extras - boto + botocore Pillow != 3.0.0 leveldb -rtests/requirements.txt +passenv = + S3_TEST_FILE_URI + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests}