Adding GCSFeedStorage

scrapy/extensions/feedexport.py

@@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
+
+class GCSFeedStorage(BlockingFeedStorage):
+
+    project_id = None
+    bucket_name = None
+    blob_name = None
+
+    def __init__(self, uri, project_id):
+        self.project_id = project_id
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):

tests/test_feedexport.py

@@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri
 import scrapy
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
-    IFeedStorage, FileFeedStorage, FTPFeedStorage,
+    IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
@@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(content, expected_content)
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    @mock.patch('scrapy.conf.settings',
+                new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123' }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client, Bucket, Blob
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gcs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        with mock.patch('google.cloud.storage.Client') as m:
+            client_mock = mock.create_autospec(Client)
+            m.return_value = client_mock
+
+            bucket_mock = mock.create_autospec(Bucket)
+            client_mock.get_bucket.return_value = bucket_mock
+
+            blob_mock = mock.create_autospec(Blob)
+            bucket_mock.blob.return_value = blob_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f)
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks