From a70ec30e19eff13d3bf18e5c392612db018d1b4f Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Wed, 1 Mar 2017 23:02:42 +0100 Subject: [PATCH 1/3] Adding new options. --- scrapy/pipelines/files.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..7041a0a78 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -40,7 +40,6 @@ class FileException(Exception): class FSFilesStore(object): - def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -79,9 +78,12 @@ class FSFilesStore(object): class S3FilesStore(object): - AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_ENDPOINT_URL = None + AWS_REGION_NAME = None + AWS_USE_SSL = None + AWS_VERIFY = None POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in # FilesPipeline.from_settings. @@ -95,8 +97,14 @@ class S3FilesStore(object): import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY) + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection @@ -181,7 +189,7 @@ class S3FilesStore(object): 'X-Amz-Grant-Read': 'GrantRead', 'X-Amz-Grant-Read-ACP': 'GrantReadACP', 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', - }) + }) extra = {} for key, value in six.iteritems(headers): try: @@ -226,7 +234,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -256,6 +264,10 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] store_uri = settings['FILES_STORE'] @@ -423,4 +435,5 @@ class FilesPipeline(MediaPipeline): # deprecated def file_key(self, url): return self.file_path(url) + file_key._base = True From 605691792f5198d75524a1ad952489193970d4a7 Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:35:39 +0100 Subject: [PATCH 2/3] Updating media-pipeline docs for S3-like storage. --- docs/topics/media-pipeline.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..733a7fe2b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -169,6 +169,18 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. +Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like +self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: + + AWS_ENDPOINT_URL = 'http://minio.example.com:9000' + +For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:: + + AWS_USE_SSL = False # or True (None by default) + AWS_VERIFY = False # or True (None by default) + +.. _Minio: https://github.com/minio/minio +.. _s3.scality: https://s3.scality.com/ .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Usage example From 11cf6ad4258250b8a2410029eb794ed892dc4cea Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:48:06 +0100 Subject: [PATCH 3/3] Comments for AWS_ENDPOINT_URL setting. --- docs/topics/settings.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ccdd02c4e..3ac3fd5af 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend `. .. setting:: BOT_NAME +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. + +.. setting:: AWS_ENDPOINT_URL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. +By default SSL will be used. + +.. setting:: AWS_USE_SSL + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. + +.. setting:: AWS_VERIFY + BOT_NAME --------