Rewrite download handler tests to coroutines. (#6846)

tests/test_downloader_handler_twisted_http10.py

@@ -9,6 +9,7 @@ import pytest
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.http import Request
 from scrapy.spiders import Spider
+from scrapy.utils.defer import deferred_f_from_coro_f
 from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase
 
 if TYPE_CHECKING:
@@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin:
 class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):
     """HTTP 1.0 test case"""
 
-    def test_protocol(self):
+    @deferred_f_from_coro_f
+    async def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.protocol)
-        d.addCallback(self.assertEqual, "HTTP/1.0")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.protocol == "HTTP/1.0"
 
 
 class TestHttps10(TestHttp10):

tests/test_downloader_handler_twisted_http2.py

@@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLED
 
 from scrapy.http import Request
 from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+    deferred_f_from_coro_f,
+    maybe_deferred_to_future,
+)
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.mockserver import ssl_context_factory
@@ -50,15 +54,14 @@ class H2DownloadHandlerMixin:
 class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
-    def test_protocol(self):
+    @deferred_f_from_coro_f
+    async def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.protocol)
-        d.addCallback(self.assertEqual, "h2")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.protocol == "h2"
 
-    @defer.inlineCallbacks
-    def test_download_with_maxsize_very_large_file(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_maxsize_very_large_file(self):
         from twisted.internet import reactor
 
         with mock.patch("scrapy.core.http2.stream.logger") as logger:
@@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
             def check(logger):
                 logger.error.assert_called_once_with(mock.ANY)
 
-            d = self.download_request(request, Spider("foo", download_maxsize=1500))
-            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+            with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+                await self.download_request(
+                    request, Spider("foo", download_maxsize=1500)
+                )
 
             # As the error message is logged in the dataReceived callback, we
             # have to give a bit of time to the reactor to process the queue
@@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
             d = defer.Deferred()
             d.addCallback(check)
             reactor.callLater(0.1, d.callback, logger)
-            yield d
+            await maybe_deferred_to_future(d)
 
-    @defer.inlineCallbacks
-    def test_unsupported_scheme(self):
+    @deferred_f_from_coro_f
+    async def test_unsupported_scheme(self):
         request = Request("ftp://unsupported.scheme")
-        d = self.download_request(request, Spider("foo"))
-        yield self.assertFailure(d, SchemeNotSupported)
+        with pytest.raises(SchemeNotSupported):
+            await self.download_request(request, Spider("foo"))
 
     def test_download_broken_content_cause_data_loss(self, url="broken"):
         pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
@@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
 
-    def test_concurrent_requests_same_domain(self):
+    @deferred_f_from_coro_f
+    async def test_concurrent_requests_same_domain(self):
         spider = Spider("foo")
 
         request1 = Request(self.getURL("file"))
-        d1 = self.download_request(request1, spider)
-        d1.addCallback(lambda r: r.body)
-        d1.addCallback(self.assertEqual, b"0123456789")
+        response1 = await self.download_request(request1, spider)
+        assert response1.body == b"0123456789"
 
         request2 = Request(self.getURL("echo"), method="POST")
-        d2 = self.download_request(request2, spider)
-        d2.addCallback(lambda r: r.headers["Content-Length"])
-        d2.addCallback(self.assertEqual, b"79")
-
-        return defer.DeferredList([d1, d2])
+        response2 = await self.download_request(request2, spider)
+        assert response2.headers["Content-Length"] == b"79"
 
     @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")
-    def test_connect_request(self):
+    @deferred_f_from_coro_f
+    async def test_connect_request(self):
         request = Request(self.getURL("file"), method="CONNECT")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b""
 
-    def test_custom_content_length_good(self):
+    @deferred_f_from_coro_f
+    async def test_custom_content_length_good(self):
         request = Request(self.getURL("contentlength"))
         custom_content_length = str(len(request.body))
         request.headers["Content-Length"] = custom_content_length
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.text)
-        d.addCallback(self.assertEqual, custom_content_length)
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.text == custom_content_length
 
-    def test_custom_content_length_bad(self):
+    @deferred_f_from_coro_f
+    async def test_custom_content_length_bad(self):
         request = Request(self.getURL("contentlength"))
         actual_content_length = str(len(request.body))
         bad_content_length = str(len(request.body) + 1)
         request.headers["Content-Length"] = bad_content_length
-        log = LogCapture()
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.text)
-        d.addCallback(self.assertEqual, actual_content_length)
-        d.addCallback(
-            lambda _: log.check_present(
-                (
-                    "scrapy.core.http2.stream",
-                    "WARNING",
-                    f"Ignoring bad Content-Length header "
-                    f"{bad_content_length!r} of request {request}, sending "
-                    f"{actual_content_length!r} instead",
-                )
+        with LogCapture() as log:
+            response = await self.download_request(request, Spider("foo"))
+        assert response.text == actual_content_length
+        log.check_present(
+            (
+                "scrapy.core.http2.stream",
+                "WARNING",
+                f"Ignoring bad Content-Length header "
+                f"{bad_content_length!r} of request {request}, sending "
+                f"{actual_content_length!r} instead",
             )
         )
-        d.addCallback(lambda _: log.uninstall())
-        return d
 
-    def test_duplicate_header(self):
+    @deferred_f_from_coro_f
+    async def test_duplicate_header(self):
         request = Request(self.getURL("echo"))
         header, value1, value2 = "Custom-Header", "foo", "bar"
         request.headers.appendlist(header, value1)
         request.headers.appendlist(header, value2)
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: json.loads(r.text)["headers"][header])
-        d.addCallback(self.assertEqual, [value1, value2])
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert json.loads(response.text)["headers"][header] == [value1, value2]
 
 
 class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
@@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase):
         self.download_handler = build_from_crawler(
             self.download_handler_cls, get_crawler()
         )
-        self.download_request = self.download_handler.download_request
 
     def getURL(self, path):
         return f"{self.scheme}://{self.host}:{self.portno}/{path}"
 
-    @defer.inlineCallbacks
-    def test_download_with_proxy_https_timeout(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_proxy_https_timeout(self):
         with pytest.raises(NotImplementedError):
-            yield super().test_download_with_proxy_https_timeout()
+            await maybe_deferred_to_future(
+                super().test_download_with_proxy_https_timeout()
+            )

tests/test_downloader_handlers.py

@@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import HtmlResponse, Request
+from scrapy.http import HtmlResponse, Request, Response
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+    deferred_f_from_coro_f,
+    maybe_deferred_to_future,
+)
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
@@ -95,28 +99,33 @@ class TestFile(unittest.TestCase):
         # add a special char to check that they are handled correctly
         self.fd, self.tmpname = mkstemp(suffix="^")
         Path(self.tmpname).write_text("0123456789", encoding="utf-8")
-        handler = build_from_crawler(FileDownloadHandler, get_crawler())
-        self.download_request = handler.download_request
+        self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler())
 
     def tearDown(self):
         os.close(self.fd)
         Path(self.tmpname).unlink()
 
-    def test_download(self):
-        def _test(response):
-            assert response.url == request.url
-            assert response.status == 200
-            assert response.body == b"0123456789"
-            assert response.protocol is None
+    async def download_request(self, request: Request, spider: Spider) -> Response:
+        return await maybe_deferred_to_future(
+            self.download_handler.download_request(request, spider)
+        )
 
+    @deferred_f_from_coro_f
+    async def test_download(self):
         request = Request(path_to_file_uri(self.tmpname))
         assert request.url.upper().endswith("%5E")
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.url == request.url
+        assert response.status == 200
+        assert response.body == b"0123456789"
+        assert response.protocol is None
 
-    def test_non_existent(self):
+    @deferred_f_from_coro_f
+    async def test_non_existent(self):
         request = Request(path_to_file_uri(mkdtemp()))
-        d = self.download_request(request, Spider("foo"))
-        return self.assertFailure(d, OSError)
+        # the specific exception differs between platforms
+        with pytest.raises(OSError):  # noqa: PT011
+            await self.download_request(request, Spider("foo"))
 
 
 class HttpDownloadHandlerMock:
@@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase):
     def setUp(self):
         crawler = get_crawler()
         self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler)
-        self.download_request = self.download_handler.download_request
         self.spider = Spider("foo")
 
-    def test_response_attrs(self):
+    async def download_request(self, request: Request, spider: Spider) -> Response:
+        return await maybe_deferred_to_future(
+            self.download_handler.download_request(request, spider)
+        )
+
+    @deferred_f_from_coro_f
+    async def test_response_attrs(self):
         uri = "data:,A%20brief%20note"
-
-        def _test(response):
-            assert response.url == uri
-            assert not response.headers
-
         request = Request(uri)
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_default_mediatype_encoding(self):
-        def _test(response):
-            assert response.text == "A brief note"
-            assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
-            assert response.encoding == "US-ASCII"
+        response = await self.download_request(request, self.spider)
+        assert response.url == uri
+        assert not response.headers
 
+    @deferred_f_from_coro_f
+    async def test_default_mediatype_encoding(self):
         request = Request("data:,A%20brief%20note")
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_default_mediatype(self):
-        def _test(response):
-            assert response.text == "\u038e\u03a3\u038e"
-            assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
-            assert response.encoding == "iso-8859-7"
+        response = await self.download_request(request, self.spider)
+        assert response.text == "A brief note"
+        assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert response.encoding == "US-ASCII"
 
+    @deferred_f_from_coro_f
+    async def test_default_mediatype(self):
         request = Request("data:;charset=iso-8859-7,%be%d3%be")
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_text_charset(self):
-        def _test(response):
-            assert response.text == "\u038e\u03a3\u038e"
-            assert response.body == b"\xbe\xd3\xbe"
-            assert response.encoding == "iso-8859-7"
+        response = await self.download_request(request, self.spider)
+        assert response.text == "\u038e\u03a3\u038e"
+        assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert response.encoding == "iso-8859-7"
 
+    @deferred_f_from_coro_f
+    async def test_text_charset(self):
         request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_mediatype_parameters(self):
-        def _test(response):
-            assert response.text == "\u038e\u03a3\u038e"
-            assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
-            assert response.encoding == "utf-8"
+        response = await self.download_request(request, self.spider)
+        assert response.text == "\u038e\u03a3\u038e"
+        assert response.body == b"\xbe\xd3\xbe"
+        assert response.encoding == "iso-8859-7"
 
+    @deferred_f_from_coro_f
+    async def test_mediatype_parameters(self):
         request = Request(
             "data:text/plain;foo=%22foo;bar%5C%22%22;"
             "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22"
             ",%CE%8E%CE%A3%CE%8E"
         )
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_base64(self):
-        def _test(response):
-            assert response.text == "Hello, world."
+        response = await self.download_request(request, self.spider)
+        assert response.text == "\u038e\u03a3\u038e"
+        assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert response.encoding == "utf-8"
 
+    @deferred_f_from_coro_f
+    async def test_base64(self):
         request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")
-        return self.download_request(request, self.spider).addCallback(_test)
-
-    def test_protocol(self):
-        def _test(response):
-            assert response.protocol is None
+        response = await self.download_request(request, self.spider)
+        assert response.text == "Hello, world."
 
+    @deferred_f_from_coro_f
+    async def test_protocol(self):
         request = Request("data:,")
-        return self.download_request(request, self.spider).addCallback(_test)
+        response = await self.download_request(request, self.spider)
+        assert response.protocol is None

tests/test_downloader_handlers_http_base.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import shutil
 import sys
 from abc import ABC, abstractmethod
@@ -13,14 +14,20 @@ from unittest import mock
 import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer, error
+from twisted.internet.defer import maybeDeferred
 from twisted.protocols.policies import WrappingFactory
 from twisted.trial import unittest
 from twisted.web import resource, server, static, util
 from twisted.web._newclient import ResponseFailed
 from twisted.web.http import _DataLoss
 
-from scrapy.http import Headers, HtmlResponse, Request, TextResponse
+from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
 from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+    deferred_f_from_coro_f,
+    deferred_from_coro,
+    maybe_deferred_to_future,
+)
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
@@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC):
         self.download_handler = build_from_crawler(
             self.download_handler_cls, get_crawler()
         )
-        self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC):
     def getURL(self, path):
         return f"{self.scheme}://{self.host}:{self.portno}/{path}"
 
-    def test_download(self):
+    async def download_request(self, request: Request, spider: Spider) -> Response:
+        return await maybe_deferred_to_future(
+            self.download_handler.download_request(request, spider)
+        )
+
+    @deferred_f_from_coro_f
+    async def test_download(self):
         request = Request(self.getURL("file"))
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"0123456789")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b"0123456789"
 
-    def test_download_head(self):
+    @deferred_f_from_coro_f
+    async def test_download_head(self):
         request = Request(self.getURL("file"), method="HEAD")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b""
 
-    def test_redirect_status(self):
+    @deferred_f_from_coro_f
+    async def test_redirect_status(self):
         request = Request(self.getURL("redirect"))
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.status)
-        d.addCallback(self.assertEqual, 302)
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.status == 302
 
-    def test_redirect_status_head(self):
+    @deferred_f_from_coro_f
+    async def test_redirect_status_head(self):
         request = Request(self.getURL("redirect"), method="HEAD")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.status)
-        d.addCallback(self.assertEqual, 302)
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.status == 302
 
-    @defer.inlineCallbacks
-    def test_timeout_download_from_spider_nodata_rcvd(self):
+    @deferred_f_from_coro_f
+    async def test_timeout_download_from_spider_nodata_rcvd(self):
         if self.reactor_pytest != "default" and sys.platform == "win32":
             # https://twistedmatrix.com/trac/ticket/10279
             raise unittest.SkipTest(
@@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC):
         spider = Spider("foo")
         meta = {"download_timeout": 0.5}
         request = Request(self.getURL("wait"), meta=meta)
-        d = self.download_request(request, spider)
-        yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+        d = deferred_from_coro(self.download_request(request, spider))
+        with pytest.raises((defer.TimeoutError, error.TimeoutError)):
+            await maybe_deferred_to_future(d)
 
-    @defer.inlineCallbacks
-    def test_timeout_download_from_spider_server_hangs(self):
+    @deferred_f_from_coro_f
+    async def test_timeout_download_from_spider_server_hangs(self):
         if self.reactor_pytest != "default" and sys.platform == "win32":
             # https://twistedmatrix.com/trac/ticket/10279
             raise unittest.SkipTest(
@@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC):
         spider = Spider("foo")
         meta = {"download_timeout": 0.5}
         request = Request(self.getURL("hang-after-headers"), meta=meta)
-        d = self.download_request(request, spider)
-        yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
-
-    def test_host_header_not_in_request_headers(self):
-        def _test(response):
-            assert response.body == to_bytes(f"{self.host}:{self.portno}")
-            assert not request.headers
+        d = deferred_from_coro(self.download_request(request, spider))
+        with pytest.raises((defer.TimeoutError, error.TimeoutError)):
+            await maybe_deferred_to_future(d)
 
+    @deferred_f_from_coro_f
+    async def test_host_header_not_in_request_headers(self):
         request = Request(self.getURL("host"))
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == to_bytes(f"{self.host}:{self.portno}")
+        assert not request.headers
 
-    def test_host_header_seted_in_request_headers(self):
+    @deferred_f_from_coro_f
+    async def test_host_header_set_in_request_headers(self):
         host = self.host + ":" + str(self.portno)
-
-        def _test(response):
-            assert response.body == host.encode()
-            assert request.headers.get("Host") == host.encode()
-
         request = Request(self.getURL("host"), headers={"Host": host})
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == host.encode()
+        assert request.headers.get("Host") == host.encode()
 
-    def test_content_length_zero_bodyless_post_request_headers(self):
+    @deferred_f_from_coro_f
+    async def test_content_length_zero_bodyless_post_request_headers(self):
         """Tests if "Content-Length: 0" is sent for bodyless POST requests.
 
         This is not strictly required by HTTP RFCs but can cause trouble
@@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC):
         https://github.com/kennethreitz/requests/issues/405
         https://bugs.python.org/issue14721
         """
-
-        def _test(response):
-            assert response.body == b"0"
-
         request = Request(self.getURL("contentlength"), method="POST")
-        return self.download_request(request, Spider("foo")).addCallback(_test)
-
-    def test_content_length_zero_bodyless_post_only_one(self):
-        def _test(response):
-            import json
-
-            headers = Headers(json.loads(response.text)["headers"])
-            contentlengths = headers.getlist("Content-Length")
-            assert len(contentlengths) == 1
-            assert contentlengths == [b"0"]
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b"0"
 
+    @deferred_f_from_coro_f
+    async def test_content_length_zero_bodyless_post_only_one(self):
         request = Request(self.getURL("echo"), method="POST")
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        headers = Headers(json.loads(response.text)["headers"])
+        contentlengths = headers.getlist("Content-Length")
+        assert len(contentlengths) == 1
+        assert contentlengths == [b"0"]
 
-    def test_payload(self):
+    @deferred_f_from_coro_f
+    async def test_payload(self):
         body = b"1" * 100  # PayloadResource requires body length to be 100
         request = Request(self.getURL("payload"), method="POST", body=body)
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, body)
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == body
 
-    def test_response_header_content_length(self):
+    @deferred_f_from_coro_f
+    async def test_response_header_content_length(self):
         request = Request(self.getURL("file"), method=b"GET")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.headers[b"content-length"])
-        d.addCallback(self.assertEqual, b"159")
-        return d
-
-    def _test_response_class(self, filename, body, response_class):
-        def _test(response):
-            assert type(response) is response_class  # pylint: disable=unidiomatic-typecheck
+        response = await self.download_request(request, Spider("foo"))
+        assert response.headers[b"content-length"] == b"159"
 
+    async def _test_response_class(
+        self, filename: str, body: bytes, response_class: type[Response]
+    ) -> None:
         request = Request(self.getURL(filename), body=body)
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert type(response) is response_class  # pylint: disable=unidiomatic-typecheck
 
-    def test_response_class_from_url(self):
-        return self._test_response_class("foo.html", b"", HtmlResponse)
+    @deferred_f_from_coro_f
+    async def test_response_class_from_url(self):
+        await self._test_response_class("foo.html", b"", HtmlResponse)
 
-    def test_response_class_from_body(self):
-        return self._test_response_class(
+    @deferred_f_from_coro_f
+    async def test_response_class_from_body(self):
+        await self._test_response_class(
             "foo",
             b"<!DOCTYPE html>\n<title>.</title>",
             HtmlResponse,
         )
 
-    def test_get_duplicate_header(self):
-        def _test(response):
-            assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
-
+    @deferred_f_from_coro_f
+    async def test_get_duplicate_header(self):
         request = Request(self.getURL("duplicate-header"))
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
 
 
 class TestHttp11Base(TestHttpBase):
     """HTTP 1.1 test case"""
 
-    def test_download_without_maxsize_limit(self):
+    @deferred_f_from_coro_f
+    async def test_download_without_maxsize_limit(self):
         request = Request(self.getURL("file"))
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"0123456789")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b"0123456789"
 
-    def test_response_class_choosing_request(self):
+    @deferred_f_from_coro_f
+    async def test_response_class_choosing_request(self):
         """Tests choosing of correct response type
         in case of Content-Type is empty but body contains text.
         """
         body = b"Some plain text\ndata with tabs\t and null bytes\0"
-
-        def _test_type(response):
-            assert type(response) is TextResponse  # pylint: disable=unidiomatic-typecheck
-
         request = Request(self.getURL("nocontenttype"), body=body)
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(_test_type)
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert type(response) is TextResponse  # pylint: disable=unidiomatic-typecheck
 
-    @defer.inlineCallbacks
-    def test_download_with_maxsize(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_maxsize(self):
         request = Request(self.getURL("file"))
 
         # 10 is minimal size for this request and the limit is only counted on
         # response body. (regardless of headers)
-        d = self.download_request(request, Spider("foo", download_maxsize=10))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"0123456789")
-        yield d
+        response = await self.download_request(
+            request, Spider("foo", download_maxsize=10)
+        )
+        assert response.body == b"0123456789"
 
-        d = self.download_request(request, Spider("foo", download_maxsize=9))
-        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+        with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+            await self.download_request(request, Spider("foo", download_maxsize=9))
 
-    @defer.inlineCallbacks
-    def test_download_with_maxsize_very_large_file(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_maxsize_very_large_file(self):
         from twisted.internet import reactor
 
         # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11
@@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase):
             def check(logger):
                 logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
 
-            d = self.download_request(request, Spider("foo", download_maxsize=1500))
-            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+            with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+                await self.download_request(
+                    request, Spider("foo", download_maxsize=1500)
+                )
 
             # As the error message is logged in the dataReceived callback, we
             # have to give a bit of time to the reactor to process the queue
@@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase):
             d = defer.Deferred()
             d.addCallback(check)
             reactor.callLater(0.1, d.callback, logger)
-            yield d
+            await maybe_deferred_to_future(d)
 
-    @defer.inlineCallbacks
-    def test_download_with_maxsize_per_req(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_maxsize_per_req(self):
         meta = {"download_maxsize": 2}
         request = Request(self.getURL("file"), meta=meta)
-        d = self.download_request(request, Spider("foo"))
-        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+        with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+            await self.download_request(request, Spider("foo"))
 
-    @defer.inlineCallbacks
-    def test_download_with_small_maxsize_per_spider(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_small_maxsize_per_spider(self):
         request = Request(self.getURL("file"))
-        d = self.download_request(request, Spider("foo", download_maxsize=2))
-        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+        with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+            await self.download_request(request, Spider("foo", download_maxsize=2))
 
-    def test_download_with_large_maxsize_per_spider(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_large_maxsize_per_spider(self):
         request = Request(self.getURL("file"))
-        d = self.download_request(request, Spider("foo", download_maxsize=100))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"0123456789")
-        return d
+        response = await self.download_request(
+            request, Spider("foo", download_maxsize=100)
+        )
+        assert response.body == b"0123456789"
 
-    def test_download_chunked_content(self):
+    @deferred_f_from_coro_f
+    async def test_download_chunked_content(self):
         request = Request(self.getURL("chunked"))
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"chunked content\n")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b"chunked content\n"
 
-    def test_download_broken_content_cause_data_loss(self, url="broken"):
+    @deferred_f_from_coro_f
+    async def test_download_broken_content_cause_data_loss(
+        self, url: str = "broken"
+    ) -> None:
         # TODO: this one checks for Twisted-specific exceptions
         request = Request(self.getURL(url))
-        d = self.download_request(request, Spider("foo"))
-
-        def checkDataLoss(failure):
-            if failure.check(ResponseFailed) and any(
-                r.check(_DataLoss) for r in failure.value.reasons
-            ):
-                return None
-            return failure
-
-        d.addCallback(lambda _: self.fail("No DataLoss exception"))
-        d.addErrback(checkDataLoss)
-        return d
+        with pytest.raises(ResponseFailed) as exc_info:
+            await self.download_request(request, Spider("foo"))
+        assert any(r.check(_DataLoss) for r in exc_info.value.reasons)
 
     def test_download_broken_chunked_content_cause_data_loss(self):
         return self.test_download_broken_content_cause_data_loss("broken-chunked")
 
-    def test_download_broken_content_allow_data_loss(self, url="broken"):
+    @deferred_f_from_coro_f
+    async def test_download_broken_content_allow_data_loss(
+        self, url: str = "broken"
+    ) -> None:
         request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False})
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.flags)
-        d.addCallback(self.assertEqual, ["dataloss"])
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.flags == ["dataloss"]
 
     def test_download_broken_chunked_content_allow_data_loss(self):
         return self.test_download_broken_content_allow_data_loss("broken-chunked")
 
-    def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):
+    @deferred_f_from_coro_f
+    async def test_download_broken_content_allow_data_loss_via_setting(
+        self, url: str = "broken"
+    ) -> None:
         crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})
         download_handler = build_from_crawler(self.download_handler_cls, crawler)
         request = Request(self.getURL(url))
-        d = download_handler.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.flags)
-        d.addCallback(self.assertEqual, ["dataloss"])
-        return d
+        response = await maybe_deferred_to_future(
+            download_handler.download_request(request, Spider("foo"))
+        )
+        assert response.flags == ["dataloss"]
 
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         return self.test_download_broken_content_allow_data_loss_via_setting(
             "broken-chunked"
         )
 
-    def test_protocol(self):
+    @deferred_f_from_coro_f
+    async def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.protocol)
-        d.addCallback(self.assertEqual, "HTTP/1.1")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.protocol == "HTTP/1.1"
 
 
 class TestHttps11Base(TestHttp11Base):
@@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base):
         'subject "/C=IE/O=Scrapy/CN=localhost"'
     )
 
-    @defer.inlineCallbacks
-    def test_tls_logging(self):
+    @deferred_f_from_coro_f
+    async def test_tls_logging(self):
         crawler = get_crawler(
             settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
         )
@@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base):
         try:
             with LogCapture() as log_capture:
                 request = Request(self.getURL("file"))
-                d = download_handler.download_request(request, Spider("foo"))
-                d.addCallback(lambda r: r.body)
-                d.addCallback(self.assertEqual, b"0123456789")
-                yield d
+                response = await maybe_deferred_to_future(
+                    download_handler.download_request(request, Spider("foo"))
+                )
+                assert response.body == b"0123456789"
                 log_capture.check_present(
                     ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)
                 )
         finally:
-            yield download_handler.close()
+            await maybe_deferred_to_future(maybeDeferred(download_handler.close))
 
 
 class TestSimpleHttpsBase(unittest.TestCase, ABC):
@@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):
             settings_dict = None
         crawler = get_crawler(settings_dict=settings_dict)
         self.download_handler = build_from_crawler(self.download_handler_cls, crawler)
-        self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):
     def getURL(self, path):
         return f"https://{self.host}:{self.portno}/{path}"
 
-    def test_download(self):
+    async def download_request(self, request: Request, spider: Spider) -> Response:
+        return await maybe_deferred_to_future(
+            self.download_handler.download_request(request, spider)
+        )
+
+    @deferred_f_from_coro_f
+    async def test_download(self):
         request = Request(self.getURL("file"))
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"0123456789")
-        return d
+        response = await self.download_request(request, Spider("foo"))
+        assert response.body == b"0123456789"
 
 
 class TestHttpsWrongHostnameBase(TestSimpleHttpsBase):
@@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC):
     def tearDownClass(cls):
         cls.mockserver.__exit__(None, None, None)
 
-    @defer.inlineCallbacks
-    def test_download_with_content_length(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_content_length(self):
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
         # download it
-        yield crawler.crawl(
-            seed=Request(
-                url=self.mockserver.url("/partial", is_secure=self.is_secure),
-                meta={"download_maxsize": 1000},
+        await maybe_deferred_to_future(
+            crawler.crawl(
+                seed=Request(
+                    url=self.mockserver.url("/partial", is_secure=self.is_secure),
+                    meta={"download_maxsize": 1000},
+                )
             )
         )
         failure = crawler.spider.meta["failure"]
         assert isinstance(failure.value, defer.CancelledError)
 
-    @defer.inlineCallbacks
-    def test_download(self):
+    @deferred_f_from_coro_f
+    async def test_download(self):
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
-        yield crawler.crawl(
-            seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))
+        await maybe_deferred_to_future(
+            crawler.crawl(
+                seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))
+            )
         )
         failure = crawler.spider.meta.get("failure")
         assert failure is None
@@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC):
         self.download_handler = build_from_crawler(
             self.download_handler_cls, get_crawler()
         )
-        self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC):
     def getURL(self, path):
         return f"http://127.0.0.1:{self.portno}/{path}"
 
-    def test_download_with_proxy(self):
-        def _test(response):
-            assert response.status == 200
-            assert response.url == request.url
-            assert response.body == self.expected_http_proxy_request_body
+    async def download_request(self, request: Request, spider: Spider) -> Response:
+        return await maybe_deferred_to_future(
+            self.download_handler.download_request(request, spider)
+        )
 
+    @deferred_f_from_coro_f
+    async def test_download_with_proxy(self):
         http_proxy = self.getURL("")
         request = Request("http://example.com", meta={"proxy": http_proxy})
-        return self.download_request(request, Spider("foo")).addCallback(_test)
-
-    def test_download_without_proxy(self):
-        def _test(response):
-            assert response.status == 200
-            assert response.url == request.url
-            assert response.body == b"/path/to/resource"
+        response = await self.download_request(request, Spider("foo"))
+        assert response.status == 200
+        assert response.url == request.url
+        assert response.body == self.expected_http_proxy_request_body
 
+    @deferred_f_from_coro_f
+    async def test_download_without_proxy(self):
         request = Request(self.getURL("path/to/resource"))
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.status == 200
+        assert response.url == request.url
+        assert response.body == b"/path/to/resource"
 
-    @defer.inlineCallbacks
-    def test_download_with_proxy_https_timeout(self):
+    @deferred_f_from_coro_f
+    async def test_download_with_proxy_https_timeout(self):
         if NON_EXISTING_RESOLVABLE:
             pytest.skip("Non-existing hosts are resolvable")
         http_proxy = self.getURL("")
         domain = "https://no-such-domain.nosuch"
         request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})
-        d = self.download_request(request, Spider("foo"))
-        timeout = yield self.assertFailure(d, error.TimeoutError)
-        assert domain in timeout.osError
-
-    def test_download_with_proxy_without_http_scheme(self):
-        def _test(response):
-            assert response.status == 200
-            assert response.url == request.url
-            assert response.body == self.expected_http_proxy_request_body
+        with pytest.raises(error.TimeoutError) as exc_info:
+            await self.download_request(request, Spider("foo"))
+        assert domain in exc_info.value.osError
 
+    @deferred_f_from_coro_f
+    async def test_download_with_proxy_without_http_scheme(self):
         http_proxy = self.getURL("").replace("http://", "")
         request = Request("http://example.com", meta={"proxy": http_proxy})
-        return self.download_request(request, Spider("foo")).addCallback(_test)
+        response = await self.download_request(request, Spider("foo"))
+        assert response.status == 200
+        assert response.url == request.url
+        assert response.body == self.expected_http_proxy_request_body