From fcc6becc586e6f895cdeed66a579d5735d215de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=BCrkalp=20Burak=20KAYRANCIO=C4=9ELU?= Date: Wed, 14 Jul 2021 11:00:43 +0300 Subject: [PATCH] S3FeedStorage: allow custom endpoint (#4998) Co-authored-by: Andrey Rahmatullin --- docs/topics/feed-exports.rst | 3 ++- scrapy/extensions/feedexport.py | 7 ++++-- tests/test_feedexport.py | 44 +++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7a4b054e9..af7fce852 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -201,9 +201,10 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported feeds using this setting: +You can also define a custom ACL and custom endpoint for exported feeds using this setting: - :setting:`FEED_STORAGE_S3_ACL` +- :setting:`AWS_ENDPOINT_URL` This storage backend uses :ref:`delayed file delivery `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 89dca12f4..84a79e32d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -150,7 +150,7 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, feed_options=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.keyname = u.path[1:] # remove first "/" self.acl = acl + self.endpoint_url = endpoint_url import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) + aws_secret_access_key=self.secret_key, + endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage): access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81437b011..da0b2c786 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + endpoint_url='https://example.com' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + def test_from_crawler_without_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def test_without_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, None) + def test_from_crawler_with_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_from_crawler_with_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_ENDPOINT_URL': 'https://example.com', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() @@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl): - super().__init__(uri, access_key, secret_key, acl) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url): + super().__init__(uri, access_key, secret_key, acl, endpoint_url) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):