Merge 6db4d385d7 into d8ba1571e7

scrapy/core/downloader/handlers/s3.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
@@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler):
 
     async def download_request(self, request: Request) -> Response:
         p = urlparse_cached(request)
-        scheme = "http" if request.meta.get("is_secure") is False else "https"
+        if request.meta.get("is_secure") is False:
+            warnings.warn(
+                "Passing is_secure=False for s3:// requests is deprecated."
+                " In future Scrapy releases this flag will be ignored.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            scheme = "http"
+        else:
+            scheme = "https"
         bucket = p.hostname
         path = p.path + "?" + p.query if p.query else p.path
         url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"

tests/test_downloader_handlers.py

@@ -175,7 +175,11 @@ class TestS3Anon:
     @coroutine_test
     async def test_anon_request_insecure(self):
         req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
-        httpreq = await self.download_request(req)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing is_secure=False for s3:// requests is deprecated",
+        ):
+            httpreq = await self.download_request(req)
         assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
 
 
@@ -215,7 +219,11 @@ class TestS3:
     @coroutine_test
     async def test_insecure_opt_out(self):
         req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
-        httpreq = await self.download_request(req)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing is_secure=False for s3:// requests is deprecated",
+        ):
+            httpreq = await self.download_request(req)
         assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
 
     @coroutine_test