S3FeedStorage: allow custom endpoint (#4998)

Co-authored-by: Andrey Rahmatullin <wrar@wrar.name>

docs/topics/feed-exports.rst

@@ -201,9 +201,10 @@ passed through the following settings:
 -   :setting:`AWS_ACCESS_KEY_ID`
 -   :setting:`AWS_SECRET_ACCESS_KEY`
 
-You can also define a custom ACL for exported feeds using this setting:
+You can also define a custom ACL and custom endpoint for exported feeds using this setting:
 
 -   :setting:`FEED_STORAGE_S3_ACL`
+-   :setting:`AWS_ENDPOINT_URL`
 
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 

scrapy/extensions/feedexport.py

@@ -150,7 +150,7 @@ class FileFeedStorage:
 
 class S3FeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+    def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
                  feed_options=None):
         if not is_botocore_available():
             raise NotConfigured('missing botocore library')
@@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage):
         self.secret_key = u.password or secret_key
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
+        self.endpoint_url = endpoint_url
         import botocore.session
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
             's3', aws_access_key_id=self.access_key,
-            aws_secret_access_key=self.secret_key)
+            aws_secret_access_key=self.secret_key,
+            endpoint_url=self.endpoint_url)
         if feed_options and feed_options.get('overwrite', True) is False:
             logger.warning('S3 does not support appending to files. To '
                            'suppress this warning, remove the overwrite '
@@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage):
             access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
             secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
             acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+            endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
             feed_options=feed_options,
         )
 

tests/test_feedexport.py

@@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+    def test_init_with_endpoint_url(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+            endpoint_url='https://example.com'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, 'https://example.com')
+
     def test_from_crawler_without_acl(self):
         settings = {
             'AWS_ACCESS_KEY_ID': 'access_key',
@@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
+    def test_without_endpoint_url(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv',
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, None)
+
     def test_from_crawler_with_acl(self):
         settings = {
             'AWS_ACCESS_KEY_ID': 'access_key',
@@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+    def test_from_crawler_with_endpoint_url(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'AWS_ENDPOINT_URL': 'https://example.com',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, 'https://example.com')
+
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
         skip_if_no_boto()
@@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
 
-    def __init__(self, uri, access_key, secret_key, acl):
-        super().__init__(uri, access_key, secret_key, acl)
+    def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
+        super().__init__(uri, access_key, secret_key, acl, endpoint_url)
 
 
 class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):