diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py index 807c8c4cb..bc306aa07 100644 --- a/tests/test_downloader_handler_twisted_http10.py +++ b/tests/test_downloader_handler_twisted_http10.py @@ -9,6 +9,7 @@ import pytest from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase if TYPE_CHECKING: @@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin: class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): """HTTP 1.0 test case""" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.0" class TestHttps10(TestHttp10): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 159f403d0..e058cedae 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory @@ -50,15 +54,14 @@ class H2DownloadHandlerMixin: class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "h2") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "h2" - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor with mock.patch("scrapy.core.http2.stream.logger") as logger: @@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def check(logger): logger.error.assert_called_once_with(mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_unsupported_scheme(self): + @deferred_f_from_coro_f + async def test_unsupported_scheme(self): request = Request("ftp://unsupported.scheme") - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, SchemeNotSupported) + with pytest.raises(SchemeNotSupported): + await self.download_request(request, Spider("foo")) def test_download_broken_content_cause_data_loss(self, url="broken"): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) @@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_concurrent_requests_same_domain(self): + @deferred_f_from_coro_f + async def test_concurrent_requests_same_domain(self): spider = Spider("foo") request1 = Request(self.getURL("file")) - d1 = self.download_request(request1, spider) - d1.addCallback(lambda r: r.body) - d1.addCallback(self.assertEqual, b"0123456789") + response1 = await self.download_request(request1, spider) + assert response1.body == b"0123456789" request2 = Request(self.getURL("echo"), method="POST") - d2 = self.download_request(request2, spider) - d2.addCallback(lambda r: r.headers["Content-Length"]) - d2.addCallback(self.assertEqual, b"79") - - return defer.DeferredList([d1, d2]) + response2 = await self.download_request(request2, spider) + assert response2.headers["Content-Length"] == b"79" @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - def test_connect_request(self): + @deferred_f_from_coro_f + async def test_connect_request(self): request = Request(self.getURL("file"), method="CONNECT") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_custom_content_length_good(self): + @deferred_f_from_coro_f + async def test_custom_content_length_good(self): request = Request(self.getURL("contentlength")) custom_content_length = str(len(request.body)) request.headers["Content-Length"] = custom_content_length - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, custom_content_length) - return d + response = await self.download_request(request, Spider("foo")) + assert response.text == custom_content_length - def test_custom_content_length_bad(self): + @deferred_f_from_coro_f + async def test_custom_content_length_bad(self): request = Request(self.getURL("contentlength")) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) request.headers["Content-Length"] = bad_content_length - log = LogCapture() - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, actual_content_length) - d.addCallback( - lambda _: log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) + with LogCapture() as log: + response = await self.download_request(request, Spider("foo")) + assert response.text == actual_content_length + log.check_present( + ( + "scrapy.core.http2.stream", + "WARNING", + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", ) ) - d.addCallback(lambda _: log.uninstall()) - return d - def test_duplicate_header(self): + @deferred_f_from_coro_f + async def test_duplicate_header(self): request = Request(self.getURL("echo")) header, value1, value2 = "Custom-Header", "foo", "bar" request.headers.appendlist(header, value1) request.headers.appendlist(header, value2) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: json.loads(r.text)["headers"][header]) - d.addCallback(self.assertEqual, [value1, value2]) - return d + response = await self.download_request(request, Spider("foo")) + assert json.loads(response.text)["headers"][header] == [value1, value2] class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): @@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): with pytest.raises(NotImplementedError): - yield super().test_download_with_proxy_https_timeout() + await maybe_deferred_to_future( + super().test_download_with_proxy_https_timeout() + ) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dacadb075..09cdbaf35 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse, Request +from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -95,28 +99,33 @@ class TestFile(unittest.TestCase): # add a special char to check that they are handled correctly self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = build_from_crawler(FileDownloadHandler, get_crawler()) - self.download_request = handler.download_request + self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler()) def tearDown(self): os.close(self.fd) Path(self.tmpname).unlink() - def test_download(self): - def _test(response): - assert response.url == request.url - assert response.status == 200 - assert response.body == b"0123456789" - assert response.protocol is None + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download(self): request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.url == request.url + assert response.status == 200 + assert response.body == b"0123456789" + assert response.protocol is None - def test_non_existent(self): + @deferred_f_from_coro_f + async def test_non_existent(self): request = Request(path_to_file_uri(mkdtemp())) - d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, OSError) + # the specific exception differs between platforms + with pytest.raises(OSError): # noqa: PT011 + await self.download_request(request, Spider("foo")) class HttpDownloadHandlerMock: @@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase): def setUp(self): crawler = get_crawler() self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) - self.download_request = self.download_handler.download_request self.spider = Spider("foo") - def test_response_attrs(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_response_attrs(self): uri = "data:,A%20brief%20note" - - def _test(response): - assert response.url == uri - assert not response.headers - request = Request(uri) - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype_encoding(self): - def _test(response): - assert response.text == "A brief note" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "US-ASCII" + response = await self.download_request(request, self.spider) + assert response.url == uri + assert not response.headers + @deferred_f_from_coro_f + async def test_default_mediatype_encoding(self): request = Request("data:,A%20brief%20note") - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" + @deferred_f_from_coro_f + async def test_default_mediatype(self): request = Request("data:;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_text_charset(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert response.body == b"\xbe\xd3\xbe" - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_text_charset(self): request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_mediatype_parameters(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "utf-8" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_mediatype_parameters(self): request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22" ",%CE%8E%CE%A3%CE%8E" ) - return self.download_request(request, self.spider).addCallback(_test) - - def test_base64(self): - def _test(response): - assert response.text == "Hello, world." + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" + @deferred_f_from_coro_f + async def test_base64(self): request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") - return self.download_request(request, self.spider).addCallback(_test) - - def test_protocol(self): - def _test(response): - assert response.protocol is None + response = await self.download_request(request, self.spider) + assert response.text == "Hello, world." + @deferred_f_from_coro_f + async def test_protocol(self): request = Request("data:,") - return self.download_request(request, self.spider).addCallback(_test) + response = await self.download_request(request, self.spider) + assert response.protocol is None diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 5eaf66966..14e12a3e6 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import shutil import sys from abc import ABC, abstractmethod @@ -13,14 +14,20 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error +from twisted.internet.defer import maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_download_head(self): + @deferred_f_from_coro_f + async def test_download_head(self): request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_redirect_status(self): + @deferred_f_from_coro_f + async def test_redirect_status(self): request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - def test_redirect_status_head(self): + @deferred_f_from_coro_f + async def test_redirect_status_head(self): request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) + @deferred_f_from_coro_f + async def test_host_header_not_in_request_headers(self): request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers - def test_host_header_seted_in_request_headers(self): + @deferred_f_from_coro_f + async def test_host_header_set_in_request_headers(self): host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() - def test_content_length_zero_bodyless_post_request_headers(self): + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_request_headers(self): """Tests if "Content-Length: 0" is sent for bodyless POST requests. This is not strictly required by HTTP RFCs but can cause trouble @@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ - - def _test(response): - assert response.body == b"0" - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0" + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_only_one(self): request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] - def test_payload(self): + @deferred_f_from_coro_f + async def test_payload(self): body = b"1" * 100 # PayloadResource requires body length to be 100 request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == body - def test_response_header_content_length(self): + @deferred_f_from_coro_f + async def test_response_header_content_length(self): request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + response = await self.download_request(request, Spider("foo")) + assert response.headers[b"content-length"] == b"159" + async def _test_response_class( + self, filename: str, body: bytes, response_class: type[Response] + ) -> None: request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) + @deferred_f_from_coro_f + async def test_response_class_from_url(self): + await self._test_response_class("foo.html", b"", HtmlResponse) - def test_response_class_from_body(self): - return self._test_response_class( + @deferred_f_from_coro_f + async def test_response_class_from_body(self): + await self._test_response_class( "foo", b"\n.", HtmlResponse, ) - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - + @deferred_f_from_coro_f + async def test_get_duplicate_header(self): request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] class TestHttp11Base(TestHttpBase): """HTTP 1.1 test case""" - def test_download_without_maxsize_limit(self): + @deferred_f_from_coro_f + async def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_response_class_choosing_request(self): + @deferred_f_from_coro_f + async def test_response_class_choosing_request(self): """Tests choosing of correct response type in case of Content-Type is empty but body contains text. """ body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d + response = await self.download_request(request, Spider("foo")) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - @defer.inlineCallbacks - def test_download_with_maxsize(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize(self): request = Request(self.getURL("file")) # 10 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await self.download_request( + request, Spider("foo", download_maxsize=10) + ) + assert response.body == b"0123456789" - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=9)) - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 @@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase): def check(logger): logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_per_req(self): meta = {"download_maxsize": 2} request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo")) - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_small_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=2)) - def test_download_with_large_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_large_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request( + request, Spider("foo", download_maxsize=100) + ) + assert response.body == b"0123456789" - def test_download_chunked_content(self): + @deferred_f_from_coro_f + async def test_download_chunked_content(self): request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"chunked content\n" - def test_download_broken_content_cause_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_cause_data_loss( + self, url: str = "broken" + ) -> None: # TODO: this one checks for Twisted-specific exceptions request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d + with pytest.raises(ResponseFailed) as exc_info: + await self.download_request(request, Spider("foo")) + assert any(r.check(_DataLoss) for r in exc_info.value.reasons) def test_download_broken_chunked_content_cause_data_loss(self): return self.test_download_broken_content_cause_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss( + self, url: str = "broken" + ) -> None: request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await self.download_request(request, Spider("foo")) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss(self): return self.test_download_broken_content_allow_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss_via_setting( + self, url: str = "broken" + ) -> None: crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting( "broken-chunked" ) - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.1" class TestHttps11Base(TestHttp11Base): @@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base): 'subject "/C=IE/O=Scrapy/CN=localhost"' ) - @defer.inlineCallbacks - def test_tls_logging(self): + @deferred_f_from_coro_f + async def test_tls_logging(self): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) @@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base): try: with LogCapture() as log_capture: request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.body == b"0123456789" log_capture.check_present( ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) ) finally: - yield download_handler.close() + await maybe_deferred_to_future(maybeDeferred(download_handler.close)) class TestSimpleHttpsBase(unittest.TestCase, ABC): @@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): settings_dict = None crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): def getURL(self, path): return f"https://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): @@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks - def test_download_with_content_length(self): + @deferred_f_from_coro_f + async def test_download_with_content_length(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, + await maybe_deferred_to_future( + crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) ) ) failure = crawler.spider.meta["failure"] assert isinstance(failure.value, defer.CancelledError) - @defer.inlineCallbacks - def test_download(self): + @deferred_f_from_coro_f + async def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + await maybe_deferred_to_future( + crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) ) failure = crawler.spider.meta.get("failure") assert failure is None @@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download_with_proxy(self): http_proxy = self.getURL("") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + @deferred_f_from_coro_f + async def test_download_without_proxy(self): request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") http_proxy = self.getURL("") domain = "https://no-such-domain.nosuch" request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + with pytest.raises(error.TimeoutError) as exc_info: + await self.download_request(request, Spider("foo")) + assert domain in exc_info.value.osError + @deferred_f_from_coro_f + async def test_download_with_proxy_without_http_scheme(self): http_proxy = self.getURL("").replace("http://", "") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body