diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8e718ad5b..408160ccb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,17 +1,18 @@ +from __future__ import annotations + import asyncio from gzip import BadGzipFile from unittest import mock import pytest -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, succeed from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue @@ -29,38 +30,36 @@ class TestManagerBase(TestCase): def tearDown(self): return self.crawler.engine.close_spider(self.spider) - def _download(self, request, response=None): + async def _download( + self, request: Request, response: Response | None = None + ) -> Response | Request: """Executes downloader mw manager's download method and returns - the result (Request or Response) or raise exception in case of + the result (Request or Response) or raises exception in case of failure. """ if not response: response = Response(request.url) - def download_func(request, spider): - return response + def download_func(request: Request, spider: Spider) -> Deferred[Response]: + return succeed(response) - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() - return ret + return await maybe_deferred_to_future( + self.mwman.download(download_func, request, self.spider) + ) class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @deferred_f_from_coro_f + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = await self._download(req, resp) assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @deferred_f_from_coro_f + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = await self._download(req, resp) assert isinstance(ret, Request), f"Not redirected: {ret!r}" assert to_bytes(ret.url) == resp.headers["Location"], ( "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @deferred_f_from_coro_f + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" resp = Response( @@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase): }, ) with pytest.raises(BadGzipFile): - self._download(request=req, response=resp) + await self._download(req, resp) class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @deferred_f_from_coro_f + async def test_download_func_not_called(self): resp = Response("http://example.com/index.html") class ResponseMiddleware: @@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called -class TestProcessRequestInvalidOutput(TestManagerBase): - """Invalid return value for process_request method should raise an exception""" - - def test_invalid_process_request(self): +class TestInvalidOutput(TestManagerBase): + @deferred_f_from_coro_f + async def test_invalid_process_request(self): + """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: @@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessRequestMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessResponseInvalidOutput(TestManagerBase): - """Invalid return value for process_response method should raise an exception""" - - def test_invalid_process_response(self): + @deferred_f_from_coro_f + async def test_invalid_process_response(self): + """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: @@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessResponseMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessExceptionInvalidOutput(TestManagerBase): - """Invalid return value for process_exception method should raise an exception""" - - def test_invalid_process_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_exception(self): + """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: @@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" - def test_deferred(self): + @deferred_f_from_coro_f + async def test_deferred(self): resp = Response("http://example.com/index.html") class DeferredMiddleware: @@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @deferred_f_from_coro_f + async def test_asyncdef(self): resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): - await defer.succeed(42) + await succeed(42) return resp self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @pytest.mark.only_asyncio - def test_asyncdef_asyncio(self): + @deferred_f_from_coro_f + async def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ddc9b5206..1d671134e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from typing import Any from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.test import get_crawler @@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - def _scrape_response(self): + async def _scrape_response(self) -> Any: """Execute spider mw manager's scrape_response method and return the result. Raise exception in case of failure. """ scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response( - scrape_func, self.response, self.request, self.spider + return await maybe_deferred_to_future( + self.mwman.scrape_response( + scrape_func, self.response, self.request, self.spider + ) ) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - return results[0] class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - def test_invalid_process_spider_input(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" - def test_invalid_process_spider_output(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - def test_invalid_process_spider_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception, spider): return 1 @@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - def test_process_spider_exception_return_none(self): + @deferred_f_from_coro_f + async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, ZeroDivisionError) + with pytest.raises(ZeroDivisionError): + await self._scrape_response() class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):