diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index a601bc2aa..a609cedb3 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -3,11 +3,10 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.core.downloader.handlers.base import BaseDownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import build_from_crawler +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy import Request @@ -40,7 +39,10 @@ class S3DownloadHandler(BaseDownloadHandler): ) ) - _http_handler = build_from_crawler(HTTP11DownloadHandler, crawler) + _http_handler = build_from_crawler( + load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]), + crawler, + ) self._download_http = _http_handler.download_request async def download_request(self, request: Request) -> Response: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8e635707..eadb7740e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -156,12 +156,12 @@ class HttpDownloadHandlerMock: @pytest.mark.requires_botocore class TestS3Anon: def setup_method(self): - crawler = get_crawler() - with mock.patch( - "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler", - HttpDownloadHandlerMock, - ): - self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) + crawler = get_crawler( + settings_dict={ + "DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock}, + } + ) + self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.download_request = self.s3reqh.download_request @coroutine_test @@ -183,13 +183,10 @@ class TestS3: settings_dict={ "AWS_ACCESS_KEY_ID": "0PN5J17HBGZHT7JJ3X82", "AWS_SECRET_ACCESS_KEY": "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + "DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock}, } ) - with mock.patch( - "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler", - HttpDownloadHandlerMock, - ): - s3reqh = build_from_crawler(S3DownloadHandler, crawler) + s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.download_request = s3reqh.download_request @contextlib.contextmanager