diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 0849090b4..a895b535c 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. +If you want to use a plain function as a processor, make sure it receives +``self`` as the first argument:: + + def lowercase_processor(self, values): + for v in values: + yield v.lower() + + class MyItemLoader(ItemLoader): + name_in = lowercase_processor + +This is because whenever a function is assigned as a class variable, it becomes +a method and would be passed the instance as the the first argument when being +called. See `this answer on stackoverflow`_ for more details. + The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -143,6 +157,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 41beebe98..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -189,6 +189,8 @@ Google Cloud Storage --------------------- .. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) @@ -204,6 +206,19 @@ For information about authentication, see this `documentation`_. .. _documentation: https://cloud.google.com/docs/authentication/production +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + Usage example ============= diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ab18a727d..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -208,6 +208,10 @@ class GCSFilesStore(object): CACHE_CONTROL = 'max-age=172800' + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None + def __init__(self, uri): from google.cloud import storage client = storage.Client(project=self.GCS_PROJECT_ID) @@ -239,7 +243,8 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type=self._get_content_type(headers) + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY ) @@ -314,6 +319,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c5fc12afe..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ead511473..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 60b931f48..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path): bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted bucket.delete_blob(path) - return content, blob + return content, acl, blob def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c761bd606..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase): meta = {'foo': 'bar'} path = 'full/filename' store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) self.assertIn('last_modified', s) self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) class ItemWithFiles(Item): diff --git a/tox.ini b/tox.ini index 60ff8c15e..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore