diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f18789ab0..f13a72a0b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -117,7 +117,7 @@ Supported Storage ================= File system is currently the only officially supported storage, but there is -also (undocumented) support for storing files in `Amazon S3`_. +also support for storing files in `Amazon S3`_. .. _Amazon S3: https://aws.amazon.com/s3/ @@ -146,6 +146,30 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +Amazon S3 storage +----------------- + +.. setting:: FILES_STORE_S3_ACL +.. setting:: IMAGES_STORE_S3_ACL + +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 +bucket. Scrapy will automatically upload the files to the bucket. + +For example, this is a valid :setting:`IMAGES_STORE` value:: + + IMAGES_STORE = 's3://bucket/images' + +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_S3_ACL` and +:setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to +``private``. To make the files publicly available use the ``public-read`` +policy:: + + IMAGES_STORE_S3_ACL = 'public-read' + +For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. + +.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Usage example ============= diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7f49aacdb..7c0d8e57b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -689,15 +689,6 @@ temporary files before uploading with :ref:`FTP feed storage `. -.. setting:: FILES_STORE_S3_ACL - -FILES_STORE_S3_ACL ------------------- - -Default: ``'private'`` - -S3-specific access control policy (ACL) for S3 files store. - .. setting:: ITEM_PIPELINES ITEM_PIPELINES diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a511887b6..964541d6a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -88,6 +88,8 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b9d01e155..26165da1a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -176,6 +176,8 @@ HTTPCACHE_GZIP = False HTTPPROXY_AUTH_ENCODING = 'latin-1' +IMAGES_STORE_S3_ACL = 'private' + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {}