diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index e218a8741..dd7bce2be 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,17 +1,13 @@ -import six from six.moves.urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_unicode +from scrapy.utils.boto import is_botocore from .http import HTTPDownloadHandler -def get_s3_connection(): - try: - from boto.s3.connection import S3Connection - except ImportError: - return None +def _get_boto_connection(): + from boto.s3.connection import S3Connection class _v19_S3Connection(S3Connection): """A dummy S3Connection wrapper that doesn't do any synchronous download""" @@ -53,21 +49,9 @@ class S3DownloadHandler(object): self.anon = kw.get('anon') self._signer = None - try: + if is_botocore(): import botocore.auth import botocore.credentials - except ImportError: - if six.PY3: - raise NotConfigured("missing botocore library") - _S3Connection = get_s3_connection() - if _S3Connection is None: - raise NotConfigured("missing botocore or boto library") - try: - self.conn = _S3Connection( - aws_access_key_id, aws_secret_access_key, **kw) - except Exception as ex: - raise NotConfigured(str(ex)) - else: kw.pop('anon', None) if kw: raise TypeError('Unexpected keyword arguments: %s' % kw) @@ -75,6 +59,13 @@ class S3DownloadHandler(object): SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( aws_access_key_id, aws_secret_access_key)) + else: + _S3Connection = _get_boto_connection() + try: + self.conn = _S3Connection( + aws_access_key_id, aws_secret_access_key, **kw) + except Exception as ex: + raise NotConfigured(str(ex)) self._download_http = httpdownloadhandler(settings).download_request diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fce5e251b..3dab2d77e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import without_none_values +from scrapy.utils.boto import is_botocore logger = logging.getLogger(__name__) @@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri): from scrapy.conf import settings - try: - import boto - except ImportError: - raise NotConfigured - self.connect_s3 = boto.connect_s3 u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or settings['AWS_ACCESS_KEY_ID'] self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY'] - self.keyname = u.path + self.is_botocore = is_botocore() + self.keyname = u.path[1:] # remove first "/" + if self.is_botocore: + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key) + else: + import boto + self.connect_s3 = boto.connect_s3 def _store_in_thread(self, file): file.seek(0) - conn = self.connect_s3(self.access_key, self.secret_key) - bucket = conn.get_bucket(self.bucketname, validate=False) - key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) - key.close() + if self.is_botocore: + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file) + else: + conn = self.connect_s3(self.access_key, self.secret_key) + bucket = conn.get_bucket(self.bucketname, validate=False) + key = bucket.new_key(self.keyname) + key.set_contents_from_file(file) + key.close() class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py new file mode 100644 index 000000000..fd6b22b88 --- /dev/null +++ b/scrapy/utils/boto.py @@ -0,0 +1,20 @@ +"""Boto/botocore helpers""" + +import six + +from scrapy.exceptions import NotConfigured + + +def is_botocore(): + try: + import botocore + return True + except ImportError: + if six.PY2: + try: + import boto + return False + except ImportError: + raise NotConfigured('missing botocore or boto library') + else: + raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 1ac2e575f..d2f7c0ae4 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -5,9 +5,11 @@ This module contains some assorted functions used in tests import os from importlib import import_module -import six from twisted.trial.unittest import SkipTest +from scrapy.exceptions import NotConfigured +from scrapy.utils.boto import is_botocore + def assert_aws_environ(): """Asserts the current environment is suitable for running AWS testsi. @@ -19,15 +21,9 @@ def assert_aws_environ(): def skip_if_no_boto(): try: - import botocore - except ImportError: - if six.PY2: - try: - import boto - except ImportError: - raise SkipTest('missing botocore or boto library') - else: - raise SkipTest('missing botocore library') + is_botocore() + except NotConfigured as e: + raise SkipTest(e.message) def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8015b0320..beb800fb5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.utils.test import assert_aws_environ from scrapy.utils.python import to_native_str +from scrapy.utils.boto import is_botocore class FileFeedStorageTest(unittest.TestCase): @@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase): uri = os.environ.get('FEEDTEST_S3_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") - from boto import connect_s3 storage = S3FeedStorage(uri) verifyObject(IFeedStorage, storage) file = storage.open(scrapy.Spider("default")) file.write("content") yield storage.store(file) u = urlparse(uri) - bucket = connect_s3().get_bucket(u.hostname, validate=False) - key = bucket.get_key(u.path) - self.assertEqual(key.get_contents_as_string(), "content") - bucket.delete_key(u.path) + content = self._get_content_and_delete(u.hostname, u.path[1:]) + self.assertEqual(content, "content") + + def _get_content_and_delete(self, bucket, path): + if is_botocore(): + import botocore.session + session = botocore.session.get_session() + client = session.create_client('s3') + key = client.get_object(Bucket=bucket, Key=path) + content = key['Body'].read() + client.delete_object(Bucket=bucket, Key=path) + else: + from boto import connect_s3 + bucket = connect_s3().get_bucket(bucket, validate=False) + key = bucket.get_key(path) + content = key.get_contents_as_string() + bucket.delete_key(path) + return content class StdoutFeedStorageTest(unittest.TestCase):