expose AWS_REGION_NAME in S3FeedStorage

docs/topics/feed-exports.rst

@@ -210,10 +210,12 @@ passed through the following settings:
 
 .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
 
-You can also define a custom ACL and custom endpoint for exported feeds using this setting:
+You can also define a custom ACL, custom endpoint, and region name for exported
+feeds using these settings:
 
 -   :setting:`FEED_STORAGE_S3_ACL`
 -   :setting:`AWS_ENDPOINT_URL`
+-   :setting:`AWS_REGION_NAME`
 
 The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
 storage backend is: ``True``.

scrapy/extensions/feedexport.py

@@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage):
         secret_key=None,
         acl=None,
         endpoint_url=None,
+        region_name=None,
         *,
         feed_options=None,
         session_token=None,
@@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
+        self.region_name = region_name
 
         if IS_BOTO3_AVAILABLE:
             import boto3.session
@@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 aws_secret_access_key=self.secret_key,
                 aws_session_token=self.session_token,
                 endpoint_url=self.endpoint_url,
+                region_name=self.region_name,
             )
         else:
             warnings.warn(
@@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 aws_secret_access_key=self.secret_key,
                 aws_session_token=self.session_token,
                 endpoint_url=self.endpoint_url,
+                region_name=self.region_name,
             )
 
         if feed_options and feed_options.get("overwrite", True) is False:
@@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage):
             session_token=crawler.settings["AWS_SESSION_TOKEN"],
             acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
             endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
+            region_name=crawler.settings["AWS_REGION_NAME"] or None,
             feed_options=feed_options,
         )
 

tests/test_feedexport.py

@@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.endpoint_url, "https://example.com")
 
+    def test_init_with_region_name(self):
+        region_name = "ap-east-1"
+        storage = S3FeedStorage(
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
+            region_name=region_name,
+        )
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.region_name, region_name)
+        self.assertEqual(storage.s3_client._client_config.region_name, region_name)
+
     def test_from_crawler_without_acl(self):
         settings = {
             "AWS_ACCESS_KEY_ID": "access_key",
@@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.endpoint_url, None)
 
+    def test_without_region_name(self):
+        settings = {
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            "s3://mybucket/export.csv",
+        )
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1")
+
     def test_from_crawler_with_acl(self):
         settings = {
             "AWS_ACCESS_KEY_ID": "access_key",
@@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.endpoint_url, "https://example.com")
 
+    def test_from_crawler_with_region_name(self):
+        region_name = "ap-east-1"
+        settings = {
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
+            "AWS_REGION_NAME": region_name,
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.region_name, region_name)
+        self.assertEqual(storage.s3_client._client_config.region_name, region_name)
+
     @defer.inlineCallbacks
     def test_store_without_acl(self):
         storage = S3FeedStorage(