RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Dynamic loading for S3 HTTPS handler (#7370)

This commit is contained in:
Kailesh 2026-03-30 13:42:43 +05:30 committed by GitHub
parent 31bf7c3892
commit 86a7ceaa9f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/handlers/s3.py
View File

@ -3,11 +3,10 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy import Request
@ -40,7 +39,10 @@ class S3DownloadHandler(BaseDownloadHandler):
)
)
_http_handler = build_from_crawler(HTTP11DownloadHandler, crawler)
_http_handler = build_from_crawler(
load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]),
crawler,
)
self._download_http = _http_handler.download_request
async def download_request(self, request: Request) -> Response:

19
tests/test_downloader_handlers.py
View File

@ -156,12 +156,12 @@ class HttpDownloadHandlerMock:
@pytest.mark.requires_botocore
class TestS3Anon:
def setup_method(self):
crawler = get_crawler()
with mock.patch(
"scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler",
HttpDownloadHandlerMock,
):
self.s3reqh = build_from_crawler(S3DownloadHandler, crawler)
crawler = get_crawler(
settings_dict={
"DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock},
}
)
self.s3reqh = build_from_crawler(S3DownloadHandler, crawler)
self.download_request = self.s3reqh.download_request
@coroutine_test
@ -183,13 +183,10 @@ class TestS3:
settings_dict={
"AWS_ACCESS_KEY_ID": "0PN5J17HBGZHT7JJ3X82",
"AWS_SECRET_ACCESS_KEY": "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o",
"DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock},
}
)
with mock.patch(
"scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler",
HttpDownloadHandlerMock,
):
s3reqh = build_from_crawler(S3DownloadHandler, crawler)
s3reqh = build_from_crawler(S3DownloadHandler, crawler)
self.download_request = s3reqh.download_request
@contextlib.contextmanager