diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 24d69040c..0dc73c513 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -194,6 +194,27 @@ You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` +.. _topics-feed-storage-gcs: + +Google Cloud Storage (GCS) +-------------------------- + +The feeds are stored on `Google Cloud Storage`_. + + * URI scheme: ``gs`` + * Example URIs: + + * ``gs://mybucket/path/to/export.csv`` + + * Required external libraries: `google-cloud-storage `_. + +For more information about authentication, please refer to `Google Cloud documentation `_. + +You can set a *Project ID* and *Access Control List (ACL)* through the following settings: + + * :setting:`FEED_STORAGE_GCS_ACL` + * :setting:`GCS_PROJECT_ID` + .. _topics-feed-storage-stdout: Standard output @@ -429,3 +450,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 9f2a06dd7..1f995ce14 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -207,7 +207,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S Google Cloud Storage --------------------- -.. setting:: GCS_PROJECT_ID .. setting:: FILES_STORE_GCS_ACL .. setting:: IMAGES_STORE_GCS_ACL diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8cc8806a5..722ae4593 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -786,6 +786,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FEED_STORAGE_GCS_ACL + +FEED_STORAGE_GCS_ACL +-------------------- + +The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -825,6 +833,15 @@ Default: ``"anonymous"`` The username to use for FTP connections when there is no ``"ftp_user"`` in ``Request`` meta. +.. setting:: GCS_PROJECT_ID + +GCS_PROJECT_ID +----------------- + +Default: ``None`` + +The Project ID that will be used when storing data on `Google Cloud Storage`_. + .. setting:: ITEM_PIPELINES ITEM_PIPELINES @@ -1544,3 +1561,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search +.. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 61dad8726..72a34ae0d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -153,6 +153,32 @@ class S3FeedStorage(BlockingFeedStorage): key.close() +class GCSFeedStorage(BlockingFeedStorage): + + def __init__(self, uri, project_id, acl): + self.project_id = project_id + self.acl = acl + u = urlparse(uri) + self.bucket_name = u.hostname + self.blob_name = u.path[1:] # remove first "/" + + @classmethod + def from_crawler(cls, crawler, uri): + return cls( + uri, + crawler.settings['GCS_PROJECT_ID'], + crawler.settings['FEED_STORAGE_GCS_ACL'] or None + ) + + def _store_in_thread(self, file): + file.seek(0) + from google.cloud.storage import Client + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file, predefined_acl=self.acl) + + class FTPFeedStorage(BlockingFeedStorage): def __init__(self, uri, use_active_mode=False): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 077317c81..2fd1abbc1 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,9 +142,10 @@ FEED_STORAGES = {} FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', - 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', - 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', + 'gs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { @@ -159,6 +160,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FEED_STORAGE_FTP_ACTIVE = False +FEED_STORAGE_GCS_ACL = '' FEED_STORAGE_S3_ACL = '' FILES_STORE_S3_ACL = 'private' @@ -168,6 +170,8 @@ FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' FTP_PASSIVE_MODE = True +GCS_PROJECT_ID = None + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index faac0b12f..7442a2f33 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,10 +2,10 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import -from posixpath import split import asyncio import os +from posixpath import split +from unittest import mock from importlib import import_module from twisted.trial.unittest import SkipTest @@ -126,3 +126,19 @@ def get_from_asyncio_queue(value): getter = q.get() q.put_nowait(value) return getter + + +def mock_google_cloud_storage(): + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Client, Bucket, Blob + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f7b997560..37384081a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -25,11 +25,23 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner from scrapy.exporters import CsvItemExporter -from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage, - IFeedStorage, S3FeedStorage, StdoutFeedStorage) +from scrapy.extensions.feedexport import ( + BlockingFeedStorage, + FileFeedStorage, + FTPFeedStorage, + GCSFeedStorage, + IFeedStorage, + S3FeedStorage, + StdoutFeedStorage, +) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete +from scrapy.utils.test import ( + assert_aws_environ, + get_s3_content_and_delete, + get_crawler, + mock_google_cloud_storage, +) from tests.mockserver import MockServer @@ -364,6 +376,63 @@ class S3FeedStorageTest(unittest.TestCase): ) +class GCSFeedStorageTest(unittest.TestCase): + + def test_parse_settings(self): + try: + from google.cloud.storage import Client # noqa + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.project_id == '123' + assert storage.acl == 'publicRead' + assert storage.bucket_name == 'mybucket' + assert storage.blob_name == 'export.csv' + + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client # noqa + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + + @defer.inlineCallbacks + def test_store(self): + try: + from google.cloud.storage import Client # noqa + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + uri = 'gs://mybucket/export.csv' + project_id = 'myproject-123' + acl = 'publicRead' + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch('google.cloud.storage.Client') as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + yield storage.store(f) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with('mybucket') + bucket_mock.blob.assert_called_once_with('export.csv') + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks diff --git a/tox.ini b/tox.ini index f729327ca..5d79739bb 100644 --- a/tox.ini +++ b/tox.ini @@ -81,6 +81,7 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 + google-cloud-storage==1.29.0 Pillow==3.4.2 [testenv:pinned]