diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2218f45e7..700775e4b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -210,10 +210,12 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys -You can also define a custom ACL and custom endpoint for exported feeds using this setting: +You can also define a custom ACL, custom endpoint, and region name for exported +feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +- :setting:`AWS_REGION_NAME` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c81f396cb..84724640b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage): secret_key=None, acl=None, endpoint_url=None, + region_name=None, *, feed_options=None, session_token=None, @@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url + self.region_name = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) else: warnings.warn( @@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) if feed_options and feed_options.get("overwrite", True) is False: @@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage): session_token=crawler.settings["AWS_SESSION_TOKEN"], acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, + region_name=crawler.settings["AWS_REGION_NAME"] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 96bb688ab..9e4fb53bf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + def test_from_crawler_without_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + def test_from_crawler_with_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + @defer.inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage(