From d71a0634039d637dc10509ebb63fa2f4ef595ebb Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 12 Sep 2017 18:30:15 +0900 Subject: [PATCH 1/4] Support for Google Cloud Storage --- scrapy/pipelines/files.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index eae03752a..304d89bcf 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -194,6 +194,41 @@ class S3FilesStore(object): return extra +class GCSFilesStore(object): + + GCS_PROJECT_ID = None + + CACHE_CONTROL = 'max-age=172800' + + def __init__(self, uri): + from google.cloud import storage + client = storage.Client(project=self.GCS_PROJECT_ID) + bucket, prefix = uri[5:].split('/', 1) + self.bucket = client.bucket(bucket) + self.prefix = prefix + + def stat_file(self, path, info): + def _onsuccess(blob): + if blob: + checksum = blob.md5_hash + last_modified = time.mktime(blob.updated.timetuple()) + return {'checksum': checksum, 'last_modified': last_modified} + else: + return {} + + return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + + def persist_file(self, path, buf, info, meta=None, headers=None): + blob = self.bucket.blob(self.prefix + path) + blob.cache_control = self.CACHE_CONTROL + blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + return threads.deferToThread( + blob.upload_from_string, + data=buf.getvalue(), + content_type='application/octet-stream' + ) + + class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading @@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline): '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, + 'gs': GCSFilesStore, } DEFAULT_FILES_URLS_FIELD = 'file_urls' DEFAULT_FILES_RESULT_FIELD = 'files' @@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) From e5d4364b2a0e7ae205605905ac0c5ac6fd8d15db Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 16:24:04 +0900 Subject: [PATCH 2/4] Add tests for GCS Storage --- scrapy/utils/test.py | 15 +++++++++++++++ tests/test_pipeline_files.py | 28 +++++++++++++++++++++++++++- tox.ini | 3 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d2ef68912..60b931f48 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,6 +20,12 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") + +def assert_gcs_environ(): + if 'GCS_PROJECT_ID' not in os.environ: + raise SkipTest("GCS_PROJECT_ID not found") + + def skip_if_no_boto(): try: is_botocore() @@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content +def get_gcs_content_and_delete(bucket, path): + from google.cloud import storage + client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + bucket.delete_blob(path) + return content, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e3ec04b8d..c761bd606 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,12 +11,13 @@ from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete +from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore from tests import mock @@ -375,6 +376,31 @@ class TestS3FilesStore(unittest.TestCase): self.assertEqual(key.content_type, 'image/png') +class TestGCSFilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_gcs_environ() + uri = os.environ.get('GCS_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No GCS URI available for testing") + data = b"TestGCSFilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = 'full/filename' + store = GCSFilesStore(uri) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') + u = urlparse(uri) + content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + self.assertEqual(content, data) + self.assertEqual(blob.metadata, {'foo': 'bar'}) + self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) + self.assertEqual(blob.content_type, 'application/octet-stream') + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tox.ini b/tox.ini index c7e1e43c9..0608693ba 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ deps = -rrequirements.txt # Extras botocore + google-cloud-storage Pillow != 3.0.0 leveldb -rtests/requirements.txt @@ -18,6 +19,8 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + GCS_TEST_FILE_URI + GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} From ee166ec44f38da7f5b99c6c164a7c6ff02b37c16 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 17:35:46 +0900 Subject: [PATCH 3/4] Support for ImagesPipeline --- scrapy/pipelines/files.py | 8 +++++++- scrapy/pipelines/images.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 304d89bcf..7fdb8a086 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -218,6 +218,12 @@ class GCSFilesStore(object): return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + def _get_content_type(self, headers): + if headers and 'Content-Type' in headers: + return headers['Content-Type'] + else: + return 'application/octet-stream' + def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL @@ -225,7 +231,7 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type='application/octet-stream' + content_type=self._get_content_type(headers) ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bc449431f..c5fc12afe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) From d4555b2bcc387292e5fd5bd8321c946e2e374fb7 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 29 Sep 2017 12:07:29 +0900 Subject: [PATCH 4/4] update docs for supporting google cloud storage --- docs/topics/media-pipeline.rst | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index e948913a4..9580a15d9 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket) +* Specifying where to store the media (filesystem directory, Amazon S3 bucket, + Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: @@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: Supported Storage ================= -File system is currently the only officially supported storage, but there is -also support for storing files in `Amazon S3`_. +File system is currently the only officially supported storage, but there are +also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_. .. _Amazon S3: https://aws.amazon.com/s3/ +.. _Google Cloud Storage: https://cloud.google.com/storage/ File system storage ------------------- @@ -171,6 +173,25 @@ For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Google Cloud Storage +--------------------- + +.. setting:: GCS_PROJECT_ID + +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage +bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + +For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:: + + IMAGES_STORE = 'gs://bucket/images/' + GCS_PROJECT_ID = 'project_id' + +For information about authentication, see this `documentation`_. + +.. _documentation: https://cloud.google.com/docs/authentication/production + Usage example =============