diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..cf70b8aca 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,10 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +You can also define a custom ACL for exported feeds using this setting: + + * :setting:`FEED_STORAGE_S3_ACL` + .. _topics-feed-storage-stdout: Standard output @@ -205,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_URI` (mandatory) * :setting:`FEED_FORMAT` * :setting:`FEED_STORAGES` + * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` @@ -302,6 +307,17 @@ Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. +.. setting:: FEED_STORAGE_S3_ACL + +FEED_STORAGE_S3_ACL +------------------- + +Default: ``''`` (empty string) + +A string containing a custom ACL for feeds exported to Amazon S3 by your project. + +For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs. + .. setting:: FEED_STORAGES_BASE FEED_STORAGES_BASE @@ -366,3 +382,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3b4d809e8..b2f7267a2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -93,7 +93,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None): # BEGIN Backward compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None @@ -118,6 +118,7 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" + self.acl = acl if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -130,19 +131,26 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY']) + return cls( + uri=uri, + access_key=crawler.settings['AWS_ACCESS_KEY_ID'], + secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + ) def _store_in_thread(self, file): file.seek(0) if self.is_botocore: + kwargs = {'ACL': self.acl} if self.acl else {} self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file) + Bucket=self.bucketname, Key=self.keyname, Body=file, + **kwargs) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) + kwargs = {'policy': self.acl} if self.acl else {} + key.set_contents_from_file(file, **kwargs) key.close() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..a800d39ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,6 +158,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_S3_ACL = '' + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b254b9f38..eef0384cf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -186,6 +186,151 @@ class S3FeedStorageTest(unittest.TestCase): content = get_s3_content_and_delete(u.hostname, u.path[1:]) self.assertEqual(content, expected_content) + def test_init_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_init_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_from_crawler_without_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_from_crawler_with_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_STORAGE_S3_ACL': 'custom-acl', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + @defer.inlineCallbacks + def test_store_botocore_without_acl(self): + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') + + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1]) + + @defer.inlineCallbacks + def test_store_botocore_with_acl(self): + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') + + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertEqual( + storage.s3_client.put_object.call_args[1].get('ACL'), + 'custom-acl' + ) + + @defer.inlineCallbacks + def test_store_not_botocore_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + yield storage.store(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertNotIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + + @defer.inlineCallbacks + def test_store_not_botocore_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + yield storage.store(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + class StdoutFeedStorageTest(unittest.TestCase):