diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9b7ceeb4c..1fba7f019 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,10 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy.core.downloader.handlers.base import BaseDownloadHandler -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler): async def download_request(self, request: Request) -> Response: p = urlparse_cached(request) - scheme = "http" if request.meta.get("is_secure") is False else "https" + if request.meta.get("is_secure") is False: + warnings.warn( + "Passing is_secure=False for s3:// requests is deprecated." + " In future Scrapy releases this flag will be ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + scheme = "http" + else: + scheme = "https" bucket = p.hostname path = p.path + "?" + p.query if p.query else p.path url = f"{scheme}://{bucket}.s3.amazonaws.com{path}" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4af5ca0e8..8cb9308ff 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -175,7 +175,11 @@ class TestS3Anon: @coroutine_test async def test_anon_request_insecure(self): req = Request("s3://aws-publicdatasets/", meta={"is_secure": False}) - httpreq = await self.download_request(req) + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing is_secure=False for s3:// requests is deprecated", + ): + httpreq = await self.download_request(req) assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/" @@ -215,7 +219,11 @@ class TestS3: @coroutine_test async def test_insecure_opt_out(self): req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False}) - httpreq = await self.download_request(req) + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing is_secure=False for s3:// requests is deprecated", + ): + httpreq = await self.download_request(req) assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg" @coroutine_test