Switch from `is_boto3_available()` to `IS_BOTO3_AVAILABLE` var

This commit is contained in:
jazzthief 2023-03-16 17:24:11 +01:00
parent a17d996da2
commit 4ebc08ef10
No known key found for this signature in database
GPG Key ID: 650DC0A0E6B6947C
3 changed files with 14 additions and 17 deletions

View File

@ -21,7 +21,7 @@ from zope.interface import Interface, implementer
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.boto import is_boto3_available, is_botocore_available
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
try:
import boto3 # noqa: F401
IS_BOTO3_AVAILABLE = True
except ImportError:
IS_BOTO3_AVAILABLE = False
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
self._using_boto3 = is_boto3_available()
if self._using_boto3:
if IS_BOTO3_AVAILABLE:
import boto3.session
session = boto3.session.Session()
@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file):
file.seek(0)
if self._using_boto3:
if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs

View File

@ -8,12 +8,3 @@ def is_botocore_available():
return True
except ImportError:
return False
def is_boto3_available():
try:
import boto3 # noqa: F401
return True
except ImportError:
return False

View File

@ -35,6 +35,7 @@ import scrapy
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import (
IS_BOTO3_AVAILABLE,
BlockingFeedStorage,
FeedExporter,
FileFeedStorage,
@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import (
S3FeedStorage,
StdoutFeedStorage,
_FeedSlot,
is_boto3_available,
)
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase):
file = mock.MagicMock()
if is_boto3_available():
if IS_BOTO3_AVAILABLE:
storage.s3_client = mock.MagicMock()
yield storage.store(file)
self.assertEqual(
@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
if is_boto3_available():
if IS_BOTO3_AVAILABLE:
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
if is_boto3_available():
if IS_BOTO3_AVAILABLE:
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
else:
acl = storage.s3_client.put_object.call_args[1]["ACL"]