diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 83d04e6ca..762b0fdf1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,14 +11,20 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, cast from itemadapter import ItemAdapter from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + GCSFilesStore, + S3FilesStore, + _md5sum, +) from scrapy.utils.defer import ensure_awaitable from scrapy.utils.python import to_bytes @@ -33,6 +39,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + from scrapy.settings import BaseSettings class ImageException(FileException): @@ -126,6 +133,20 @@ class ImagesPipeline(FilesPipeline): ) -> str: return await self.image_downloaded(response, request, info, item=item) + @classmethod + def _update_stores(cls, settings: BaseSettings) -> None: + super()._update_stores(settings) + + s3store: type[S3FilesStore] = cast( + "type[S3FilesStore]", cls.STORE_SCHEMES["s3"] + ) + s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] + + gcs_store: type[GCSFilesStore] = cast( + "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"] + ) + gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None + async def image_downloaded( self, response: Response, diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 38662348f..1b73dd157 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -14,6 +14,7 @@ from itemadapter import ItemAdapter from scrapy.http import Request, Response from scrapy.item import Field, Item +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler @@ -541,6 +542,44 @@ class TestImagesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value + def test_images_store_s3_acl_setting_used(self, tmp_path): + old_policy = S3FilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_S3_ACL": "public-read", + "FILES_STORE_S3_ACL": "private", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert S3FilesStore.POLICY == "public-read" + finally: + S3FilesStore.POLICY = old_policy + + def test_images_store_gcs_acl_setting_used(self, tmp_path): + old_policy = GCSFilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_GCS_ACL": "authenticatedRead", + "FILES_STORE_GCS_ACL": "", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert GCSFilesStore.POLICY == "authenticatedRead" + finally: + GCSFilesStore.POLICY = old_policy + def _create_image(format_, *a, **kw): buf = io.BytesIO()