Use boto3 session and client

This commit is contained in:
Yegor 2023-02-15 12:18:28 +01:00
parent 60bf56b715
commit 01ad49515d
No known key found for this signature in database
GPG Key ID: 25A634504C9CCE9E
1 changed files with 29 additions and 8 deletions

View File

@ -173,16 +173,37 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
import botocore.session
if is_boto3_available():
import boto3.session
session = boto3.session.Session()
self.s3_client = session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
else:
warnings.warn(
"Botocore usage is deprecated for S3FeedStorage, "
"please use boto3 to avoid problems",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
import botocore.session
session = botocore.get_session()
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
"S3 does not support appending to files. To "