Merge pull request #2923 from rhoboro/fixes-685

[MRG+2] Fixes #685 FilesPipeline support for Google Cloud Storage.

docs/topics/media-pipeline.rst

@@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
 Both pipelines implement these features:
 
 * Avoid re-downloading media that was downloaded recently
-* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
+* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
+  Google Cloud Storage bucket)
 
 The Images Pipeline has a few extra functions for processing images:
 
@@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 Supported Storage
 =================
 
-File system is currently the only officially supported storage, but there is
-also support for storing files in `Amazon S3`_.
+File system is currently the only officially supported storage, but there are
+also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
 
 .. _Amazon S3: https://aws.amazon.com/s3/
+.. _Google Cloud Storage: https://cloud.google.com/storage/
 
 File system storage
 -------------------
@@ -171,6 +173,25 @@ For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
 .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
+Google Cloud Storage
+---------------------
+
+.. setting:: GCS_PROJECT_ID
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
+bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
+
+.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
+
+    IMAGES_STORE = 'gs://bucket/images/'
+    GCS_PROJECT_ID = 'project_id'
+
+For information about authentication, see this `documentation`_.
+
+.. _documentation: https://cloud.google.com/docs/authentication/production
+
 Usage example
 =============
 

scrapy/pipelines/files.py

@@ -194,6 +194,47 @@ class S3FilesStore(object):
         return extra
 
 
+class GCSFilesStore(object):
+
+    GCS_PROJECT_ID = None
+
+    CACHE_CONTROL = 'max-age=172800'
+
+    def __init__(self, uri):
+        from google.cloud import storage
+        client = storage.Client(project=self.GCS_PROJECT_ID)
+        bucket, prefix = uri[5:].split('/', 1)
+        self.bucket = client.bucket(bucket)
+        self.prefix = prefix
+
+    def stat_file(self, path, info):
+        def _onsuccess(blob):
+            if blob:
+                checksum = blob.md5_hash
+                last_modified = time.mktime(blob.updated.timetuple())
+                return {'checksum': checksum, 'last_modified': last_modified}
+            else:
+                return {}
+
+        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+
+    def _get_content_type(self, headers):
+        if headers and 'Content-Type' in headers:
+            return headers['Content-Type']
+        else:
+            return 'application/octet-stream'
+
+    def persist_file(self, path, buf, info, meta=None, headers=None):
+        blob = self.bucket.blob(self.prefix + path)
+        blob.cache_control = self.CACHE_CONTROL
+        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        return threads.deferToThread(
+            blob.upload_from_string,
+            data=buf.getvalue(),
+            content_type=self._get_content_type(headers)
+        )
+
+
 class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
 
@@ -219,6 +260,7 @@ class FilesPipeline(MediaPipeline):
         '': FSFilesStore,
         'file': FSFilesStore,
         's3': S3FilesStore,
+        'gs': GCSFilesStore,
     }
     DEFAULT_FILES_URLS_FIELD = 'file_urls'
     DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -258,6 +300,9 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
 

scrapy/pipelines/images.py

@@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 

scrapy/utils/test.py

@@ -20,6 +20,12 @@ def assert_aws_environ():
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+
+def assert_gcs_environ():
+    if 'GCS_PROJECT_ID' not in os.environ:
+        raise SkipTest("GCS_PROJECT_ID not found")
+
+
 def skip_if_no_boto():
     try:
         is_botocore()
@@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
         bucket.delete_key(path)
     return (content, key) if with_key else content
 
+def get_gcs_content_and_delete(bucket, path):
+    from google.cloud import storage
+    client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
+    bucket = client.get_bucket(bucket)
+    blob = bucket.get_blob(path)
+    content = blob.download_as_string()
+    bucket.delete_blob(path)
+    return content, blob
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level

tests/test_pipeline_files.py

@@ -11,12 +11,13 @@ from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
 from tests import mock
@@ -375,6 +376,31 @@ class TestS3FilesStore(unittest.TestCase):
             self.assertEqual(key.content_type, 'image/png')
 
 
+class TestGCSFilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_gcs_environ()
+        uri = os.environ.get('GCS_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No GCS URI available for testing")
+        data = b"TestGCSFilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = 'full/filename'
+        store = GCSFilesStore(uri)
+        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
+        u = urlparse(uri)
+        content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        self.assertEqual(content, data)
+        self.assertEqual(blob.metadata, {'foo': 'bar'})
+        self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
+        self.assertEqual(blob.content_type, 'application/octet-stream')
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()

tox.ini

@@ -11,6 +11,7 @@ deps =
     -rrequirements.txt
     # Extras
     botocore
+    google-cloud-storage
     Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt
@@ -18,6 +19,8 @@ passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
+    GCS_TEST_FILE_URI
+    GCS_PROJECT_ID
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}