Merge branch 'master' into eagerness

docs/topics/loaders.rst

@@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator.
    containing the collected values (for that field). The result of the output
    processors is the value that will be finally assigned to the item.
 
+If you want to use a plain function as a processor, make sure it receives
+``self`` as the first argument::
+
+    def lowercase_processor(self, values):
+        for v in values:
+            yield v.lower()
+
+    class MyItemLoader(ItemLoader):
+        name_in = lowercase_processor
+
+This is because whenever a function is assigned as a class variable, it becomes
+a method and would be passed the instance as the the first argument when being
+called. See `this answer on stackoverflow`_ for more details.
+
 The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
@@ -143,6 +157,7 @@ processors to populate the fields.
 Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
+.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
 
 Declaring Item Loaders
 ======================

docs/topics/media-pipeline.rst

@@ -189,6 +189,8 @@ Google Cloud Storage
 ---------------------
 
 .. setting:: GCS_PROJECT_ID
+.. setting:: FILES_STORE_GCS_ACL
+.. setting:: IMAGES_STORE_GCS_ACL
 
 :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
 bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
@@ -204,6 +206,19 @@ For information about authentication, see this `documentation`_.
 
 .. _documentation: https://cloud.google.com/docs/authentication/production
 
+You can modify the Access Control List (ACL) policy used for the stored files,
+which is defined by the :setting:`FILES_STORE_GCS_ACL` and
+:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
+``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
+To make the files publicly available use the ``publicRead``
+policy::
+
+    IMAGES_STORE_GCS_ACL = 'publicRead'
+
+For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
+
+.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
+
 Usage example
 =============
 

scrapy/pipelines/files.py

@@ -208,6 +208,10 @@ class GCSFilesStore(object):
 
     CACHE_CONTROL = 'max-age=172800'
 
+    # The bucket's default object ACL will be applied to the object.
+    # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
+    POLICY = None
+
     def __init__(self, uri):
         from google.cloud import storage
         client = storage.Client(project=self.GCS_PROJECT_ID)
@@ -239,7 +243,8 @@ class GCSFilesStore(object):
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
-            content_type=self._get_content_type(headers)
+            content_type=self._get_content_type(headers),
+            predefined_acl=self.POLICY
         )
 
 
@@ -314,6 +319,7 @@ class FilesPipeline(MediaPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+        gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
 
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)

scrapy/pipelines/images.py

@@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+        gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None
 
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)

scrapy/settings/default_settings.py

@@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
+FILES_STORE_GCS_ACL = ''
 
 FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
@@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'
+IMAGES_STORE_GCS_ACL = ''
 
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 

scrapy/utils/test.py

@@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path):
     bucket = client.get_bucket(bucket)
     blob = bucket.get_blob(path)
     content = blob.download_as_string()
+    acl = list(blob.acl)  # loads acl before it will be deleted
     bucket.delete_blob(path)
-    return content, blob
+    return content, acl, blob
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it

tests/constraints.txt

@@ -0,0 +1 @@
+Twisted!=18.4.0

tests/test_pipeline_files.py

@@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase):
         meta = {'foo': 'bar'}
         path = 'full/filename'
         store = GCSFilesStore(uri)
+        store.POLICY = 'authenticatedRead'
+        expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'}
         yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
         s = yield store.stat_file(path, info=None)
         self.assertIn('last_modified', s)
         self.assertIn('checksum', s)
         self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
         u = urlparse(uri)
-        content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
         self.assertEqual(content, data)
         self.assertEqual(blob.metadata, {'foo': 'bar'})
         self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
         self.assertEqual(blob.content_type, 'application/octet-stream')
+        self.assertIn(expected_policy, acl)
 
 
 class ItemWithFiles(Item):

tox.ini

@@ -8,6 +8,7 @@ envlist = py27
 
 [testenv]
 deps =
+    -ctests/constraints.txt
     -rrequirements.txt
     # Extras
     botocore