Merge pull request #3608 from ejulio/feat-685

[MRG+1] Fix for #685 Add Google Cloud Storage Feed Export

docs/topics/feed-exports.rst

@@ -194,6 +194,27 @@ You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 
+.. _topics-feed-storage-gcs:
+
+Google Cloud Storage (GCS)
+--------------------------
+
+The feeds are stored on `Google Cloud Storage`_.
+
+ * URI scheme: ``gs``
+ * Example URIs:
+
+   * ``gs://mybucket/path/to/export.csv``
+
+ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+
+For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
+
+You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
+
+ * :setting:`FEED_STORAGE_GCS_ACL`
+ * :setting:`GCS_PROJECT_ID`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -429,3 +450,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+.. _Google Cloud Storage: https://cloud.google.com/storage/

docs/topics/media-pipeline.rst

@@ -207,7 +207,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 Google Cloud Storage
 ---------------------
 
-.. setting:: GCS_PROJECT_ID
 .. setting:: FILES_STORE_GCS_ACL
 .. setting:: IMAGES_STORE_GCS_ACL
 

docs/topics/settings.rst

@@ -786,6 +786,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
+.. setting:: FEED_STORAGE_GCS_ACL
+
+FEED_STORAGE_GCS_ACL
+--------------------
+
+The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
+For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+
 .. setting:: FTP_PASSIVE_MODE
 
 FTP_PASSIVE_MODE
@@ -825,6 +833,15 @@ Default: ``"anonymous"``
 The username to use for FTP connections when there is no ``"ftp_user"``
 in ``Request`` meta.
 
+.. setting:: GCS_PROJECT_ID
+
+GCS_PROJECT_ID
+-----------------
+
+Default: ``None``
+
+The Project ID that will be used when storing data on `Google Cloud Storage`_.
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -1544,3 +1561,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _Google Cloud Storage: https://cloud.google.com/storage/

scrapy/extensions/feedexport.py

@@ -153,6 +153,32 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
+class GCSFeedStorage(BlockingFeedStorage):
+
+    def __init__(self, uri, project_id, acl):
+        self.project_id = project_id
+        self.acl = acl
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(
+            uri,
+            crawler.settings['GCS_PROJECT_ID'],
+            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+        )
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file, predefined_acl=self.acl)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, use_active_mode=False):

scrapy/settings/default_settings.py

@@ -142,9 +142,10 @@ FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
+    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
@@ -159,6 +160,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FEED_STORAGE_FTP_ACTIVE = False
+FEED_STORAGE_GCS_ACL = ''
 FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
@@ -168,6 +170,8 @@ FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+GCS_PROJECT_ID = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False

scrapy/utils/test.py

@@ -2,10 +2,10 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
-from posixpath import split
 import asyncio
 import os
+from posixpath import split
+from unittest import mock
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
@@ -126,3 +126,19 @@ def get_from_asyncio_queue(value):
     getter = q.get()
     q.put_nowait(value)
     return getter
+
+
+def mock_google_cloud_storage():
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Client, Bucket, Blob
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)

tests/test_feedexport.py

@@ -25,11 +25,23 @@ from zope.interface.verify import verifyObject
 import scrapy
 from scrapy.crawler import CrawlerRunner
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
-                                          IFeedStorage, S3FeedStorage, StdoutFeedStorage)
+from scrapy.extensions.feedexport import (
+    BlockingFeedStorage,
+    FileFeedStorage,
+    FTPFeedStorage,
+    GCSFeedStorage,
+    IFeedStorage,
+    S3FeedStorage,
+    StdoutFeedStorage,
+)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete
+from scrapy.utils.test import (
+    assert_aws_environ,
+    get_s3_content_and_delete,
+    get_crawler,
+    mock_google_cloud_storage,
+)
 
 from tests.mockserver import MockServer
 
@@ -364,6 +376,63 @@ class S3FeedStorageTest(unittest.TestCase):
         )
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.acl == 'publicRead'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    def test_parse_empty_acl(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        acl = 'publicRead'
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        with mock.patch('google.cloud.storage.Client') as m:
+            m.return_value = client_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id, acl)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

tox.ini

@@ -81,6 +81,7 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
+    google-cloud-storage==1.29.0
     Pillow==3.4.2
 
 [testenv:pinned]