RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 6db4d385d7 into d8ba1571e7

This commit is contained in:
Andrey Rakhmatullin 2026-07-14 16:36:11 +00:00 committed by GitHub
parent d8ba1571e7 6db4d385d7
commit e772d5de6b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/core/downloader/handlers/s3.py
View File

@ -1,9 +1,10 @@
from __future__ import annotations from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, cast from typing import TYPE_CHECKING, Any, cast
from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.misc import build_from_crawler, load_object
@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response: async def download_request(self, request: Request) -> Response:
p = urlparse_cached(request) p = urlparse_cached(request)
scheme = "http" if request.meta.get("is_secure") is False else "https" if request.meta.get("is_secure") is False:
warnings.warn(
"Passing is_secure=False for s3:// requests is deprecated."
" In future Scrapy releases this flag will be ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
scheme = "http"
else:
scheme = "https"
bucket = p.hostname bucket = p.hostname
path = p.path + "?" + p.query if p.query else p.path path = p.path + "?" + p.query if p.query else p.path
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}" url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"

12
tests/test_downloader_handlers.py
View File

@ -175,7 +175,11 @@ class TestS3Anon:
@coroutine_test @coroutine_test
async def test_anon_request_insecure(self): async def test_anon_request_insecure(self):
req = Request("s3://aws-publicdatasets/", meta={"is_secure": False}) req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
httpreq = await self.download_request(req) with pytest.warns(
ScrapyDeprecationWarning,
match="Passing is_secure=False for s3:// requests is deprecated",
):
httpreq = await self.download_request(req)
assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/" assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
@ -215,7 +219,11 @@ class TestS3:
@coroutine_test @coroutine_test
async def test_insecure_opt_out(self): async def test_insecure_opt_out(self):
req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False}) req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
httpreq = await self.download_request(req) with pytest.warns(
ScrapyDeprecationWarning,
match="Passing is_secure=False for s3:// requests is deprecated",
):
httpreq = await self.download_request(req)
assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg" assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
@coroutine_test @coroutine_test