mirror of https://github.com/scrapy/scrapy.git
Switch from `is_boto3_available()` to `IS_BOTO3_AVAILABLE` var
This commit is contained in:
parent
a17d996da2
commit
4ebc08ef10
|
|
@ -21,7 +21,7 @@ from zope.interface import Interface, implementer
|
|||
from scrapy import Spider, signals
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_boto3_available, is_botocore_available
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
|
@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import boto3 # noqa: F401
|
||||
|
||||
IS_BOTO3_AVAILABLE = True
|
||||
except ImportError:
|
||||
IS_BOTO3_AVAILABLE = False
|
||||
|
||||
|
||||
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
||||
argument_names = get_func_args(builder)
|
||||
|
|
@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
self.keyname = u.path[1:] # remove first "/"
|
||||
self.acl = acl
|
||||
self.endpoint_url = endpoint_url
|
||||
self._using_boto3 = is_boto3_available()
|
||||
|
||||
if self._using_boto3:
|
||||
if IS_BOTO3_AVAILABLE:
|
||||
import boto3.session
|
||||
|
||||
session = boto3.session.Session()
|
||||
|
|
@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
|
||||
def _store_in_thread(self, file):
|
||||
file.seek(0)
|
||||
if self._using_boto3:
|
||||
if IS_BOTO3_AVAILABLE:
|
||||
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
|
||||
|
|
|
|||
|
|
@ -8,12 +8,3 @@ def is_botocore_available():
|
|||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def is_boto3_available():
|
||||
try:
|
||||
import boto3 # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import scrapy
|
|||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exporters import CsvItemExporter, JsonItemExporter
|
||||
from scrapy.extensions.feedexport import (
|
||||
IS_BOTO3_AVAILABLE,
|
||||
BlockingFeedStorage,
|
||||
FeedExporter,
|
||||
FileFeedStorage,
|
||||
|
|
@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import (
|
|||
S3FeedStorage,
|
||||
StdoutFeedStorage,
|
||||
_FeedSlot,
|
||||
is_boto3_available,
|
||||
)
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
|
||||
file = mock.MagicMock()
|
||||
|
||||
if is_boto3_available():
|
||||
if IS_BOTO3_AVAILABLE:
|
||||
storage.s3_client = mock.MagicMock()
|
||||
yield storage.store(file)
|
||||
self.assertEqual(
|
||||
|
|
@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
|
||||
storage.s3_client = mock.MagicMock()
|
||||
yield storage.store(BytesIO(b"test file"))
|
||||
if is_boto3_available():
|
||||
if IS_BOTO3_AVAILABLE:
|
||||
acl = (
|
||||
storage.s3_client.upload_fileobj.call_args[1]
|
||||
.get("ExtraArgs", {})
|
||||
|
|
@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
|
||||
storage.s3_client = mock.MagicMock()
|
||||
yield storage.store(BytesIO(b"test file"))
|
||||
if is_boto3_available():
|
||||
if IS_BOTO3_AVAILABLE:
|
||||
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
|
||||
else:
|
||||
acl = storage.s3_client.put_object.call_args[1]["ACL"]
|
||||
|
|
|
|||
Loading…
Reference in New Issue