diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f0a946de..83849ca13 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,7 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_boto3_available, is_botocore_available +from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +try: + import boto3 # noqa: F401 + + IS_BOTO3_AVAILABLE = True +except ImportError: + IS_BOTO3_AVAILABLE = False + def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): argument_names = get_func_args(builder) @@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - self._using_boto3 = is_boto3_available() - if self._using_boto3: + if IS_BOTO3_AVAILABLE: import boto3.session session = boto3.session.Session() @@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - if self._using_boto3: + if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 7b18b6bcf..085ee7d25 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -8,12 +8,3 @@ def is_botocore_available(): return True except ImportError: return False - - -def is_boto3_available(): - try: - import boto3 # noqa: F401 - - return True - except ImportError: - return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2e350df65..7df3e6dd3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -35,6 +35,7 @@ import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, _FeedSlot, - is_boto3_available, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase): file = mock.MagicMock() - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: storage.s3_client = mock.MagicMock() yield storage.store(file) self.assertEqual( @@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = ( storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {}) @@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] else: acl = storage.s3_client.put_object.call_args[1]["ACL"]