From 04ccf79e38561a2175c18440b3b4a53ba2f4992f Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 15:39:45 +0000 Subject: [PATCH 1/2] A different S3 Endpoint URL is now possible when uploading images --- scrapy/pipelines/images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 95323c613..8338a6281 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -89,6 +89,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] From 430e9392483b3992c16bd0314f1bcaed91a9d392 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 19:59:40 +0000 Subject: [PATCH 2/2] Added missing AWS Settings for ImagesPipeline --- scrapy/pipelines/images.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8338a6281..a1457c7e9 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -90,6 +90,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs']