diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 41beebe98..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -189,6 +189,8 @@ Google Cloud Storage --------------------- .. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) @@ -204,6 +206,19 @@ For information about authentication, see this `documentation`_. .. _documentation: https://cloud.google.com/docs/authentication/production +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + Usage example ============= diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ab18a727d..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -208,6 +208,10 @@ class GCSFilesStore(object): CACHE_CONTROL = 'max-age=172800' + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None + def __init__(self, uri): from google.cloud import storage client = storage.Client(project=self.GCS_PROJECT_ID) @@ -239,7 +243,8 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type=self._get_content_type(headers) + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY ) @@ -314,6 +319,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c5fc12afe..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ead511473..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 60b931f48..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path): bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted bucket.delete_blob(path) - return content, blob + return content, acl, blob def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c761bd606..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase): meta = {'foo': 'bar'} path = 'full/filename' store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) self.assertIn('last_modified', s) self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) class ItemWithFiles(Item):