From ee166ec44f38da7f5b99c6c164a7c6ff02b37c16 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 17:35:46 +0900 Subject: [PATCH] Support for ImagesPipeline --- scrapy/pipelines/files.py | 8 +++++++- scrapy/pipelines/images.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 304d89bcf..7fdb8a086 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -218,6 +218,12 @@ class GCSFilesStore(object): return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + def _get_content_type(self, headers): + if headers and 'Content-Type' in headers: + return headers['Content-Type'] + else: + return 'application/octet-stream' + def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL @@ -225,7 +231,7 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type='application/octet-stream' + content_type=self._get_content_type(headers) ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bc449431f..c5fc12afe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings)