From 01ad49515d31e3053949d20e923dc61a87147eab Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:18:28 +0100 Subject: [PATCH] Use boto3 session and client --- scrapy/extensions/feedexport.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cd26b5778..34eb8e4b4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -173,16 +173,37 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - import botocore.session + if is_boto3_available(): + import boto3.session + session = boto3.session.Session() + + self.s3_client = session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + else: + warnings.warn( + "Botocore usage is deprecated for S3FeedStorage, " + "please use boto3 to avoid problems", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + import botocore.session + session = botocore.get_session() session = botocore.session.get_session() - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - ) + self.s3_client = session.create_client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + if feed_options and feed_options.get("overwrite", True) is False: logger.warning( "S3 does not support appending to files. To "