RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove usages of TestCase._wait(). (#6773)

This commit is contained in:
Andrey Rakhmatullin 2025-05-05 11:44:17 +04:00 committed by GitHub
parent ff1ac75c9e
commit 2a1edbd473
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 90 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
tests/test_downloadermiddleware.py
View File

@ -1,17 +1,18 @@
from __future__ import annotations
import asyncio
from gzip import BadGzipFile
from unittest import mock
import pytest
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.internet.defer import Deferred, succeed
from twisted.trial.unittest import TestCase
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
@ -29,38 +30,36 @@ class TestManagerBase(TestCase):
def tearDown(self):
return self.crawler.engine.close_spider(self.spider)
def _download(self, request, response=None):
async def _download(
self, request: Request, response: Response | None = None
) -> Response | Request:
"""Executes downloader mw manager's download method and returns
the result (Request or Response) or raise exception in case of
the result (Request or Response) or raises exception in case of
failure.
"""
if not response:
response = Response(request.url)
def download_func(request, spider):
return response
def download_func(request: Request, spider: Spider) -> Deferred[Response]:
return succeed(response)
dfd = self.mwman.download(download_func, request, self.spider)
# catch deferred result and return the value
results = []
dfd.addBoth(results.append)
self._wait(dfd)
ret = results[0]
if isinstance(ret, Failure):
ret.raiseException()
return ret
return await maybe_deferred_to_future(
self.mwman.download(download_func, request, self.spider)
)
class TestDefaults(TestManagerBase):
"""Tests default behavior with default settings"""
def test_request_response(self):
@deferred_f_from_coro_f
async def test_request_response(self):
req = Request("http://example.com/index.html")
resp = Response(req.url, status=200)
ret = self._download(req, resp)
ret = await self._download(req, resp)
assert isinstance(ret, Response), "Non-response returned"
def test_3xx_and_invalid_gzipped_body_must_redirect(self):
@deferred_f_from_coro_f
async def test_3xx_and_invalid_gzipped_body_must_redirect(self):
"""Regression test for a failure when redirecting a compressed
request.
@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase):
"Location": "http://example.com/login",
},
)
ret = self._download(request=req, response=resp)
ret = await self._download(req, resp)
assert isinstance(ret, Request), f"Not redirected: {ret!r}"
assert to_bytes(ret.url) == resp.headers["Location"], (
"Not redirected to location header"
)
def test_200_and_invalid_gzipped_body_must_fail(self):
@deferred_f_from_coro_f
async def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com")
body = b"<p>You are being redirected</p>"
resp = Response(
@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase):
},
)
with pytest.raises(BadGzipFile):
self._download(request=req, response=resp)
await self._download(req, resp)
class TestResponseFromProcessRequest(TestManagerBase):
"""Tests middleware returning a response from process_request."""
def test_download_func_not_called(self):
@deferred_f_from_coro_f
async def test_download_func_not_called(self):
resp = Response("http://example.com/index.html")
class ResponseMiddleware:
@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase):
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
self._wait(dfd)
assert results[0] is resp
result = await maybe_deferred_to_future(
self.mwman.download(download_func, req, self.spider)
)
assert result is resp
assert not download_func.called
class TestProcessRequestInvalidOutput(TestManagerBase):
"""Invalid return value for process_request method should raise an exception"""
def test_invalid_process_request(self):
class TestInvalidOutput(TestManagerBase):
@deferred_f_from_coro_f
async def test_invalid_process_request(self):
"""Invalid return value for process_request method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessRequestMiddleware:
@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessRequestMiddleware())
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
assert isinstance(results[0], Failure)
assert isinstance(results[0].value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._download(req)
class TestProcessResponseInvalidOutput(TestManagerBase):
"""Invalid return value for process_response method should raise an exception"""
def test_invalid_process_response(self):
@deferred_f_from_coro_f
async def test_invalid_process_response(self):
"""Invalid return value for process_response method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessResponseMiddleware:
@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessResponseMiddleware())
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
assert isinstance(results[0], Failure)
assert isinstance(results[0].value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._download(req)
class TestProcessExceptionInvalidOutput(TestManagerBase):
"""Invalid return value for process_exception method should raise an exception"""
def test_invalid_process_exception(self):
@deferred_f_from_coro_f
async def test_invalid_process_exception(self):
"""Invalid return value for process_exception method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessExceptionMiddleware:
@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessExceptionMiddleware())
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
assert isinstance(results[0], Failure)
assert isinstance(results[0].value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._download(req)
class TestMiddlewareUsingDeferreds(TestManagerBase):
"""Middlewares using Deferreds should work"""
def test_deferred(self):
@deferred_f_from_coro_f
async def test_deferred(self):
resp = Response("http://example.com/index.html")
class DeferredMiddleware:
@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
self.mwman._add_middleware(DeferredMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
self._wait(dfd)
assert results[0] is resp
result = await maybe_deferred_to_future(
self.mwman.download(download_func, req, self.spider)
)
assert result is resp
assert not download_func.called
@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
class TestMiddlewareUsingCoro(TestManagerBase):
"""Middlewares using asyncio coroutines should work"""
def test_asyncdef(self):
@deferred_f_from_coro_f
async def test_asyncdef(self):
resp = Response("http://example.com/index.html")
class CoroMiddleware:
async def process_request(self, request, spider):
await defer.succeed(42)
await succeed(42)
return resp
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
self._wait(dfd)
assert results[0] is resp
result = await maybe_deferred_to_future(
self.mwman.download(download_func, req, self.spider)
)
assert result is resp
assert not download_func.called
@pytest.mark.only_asyncio
def test_asyncdef_asyncio(self):
@deferred_f_from_coro_f
async def test_asyncdef_asyncio(self):
resp = Response("http://example.com/index.html")
class CoroMiddleware:
@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase):
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
self._wait(dfd)
assert results[0] is resp
result = await maybe_deferred_to_future(
self.mwman.download(download_func, req, self.spider)
)
assert result is resp
assert not download_func.called

53
tests/test_spidermiddleware.py
View File

@ -1,12 +1,12 @@
from __future__ import annotations
from collections.abc import AsyncIterator, Iterable
from typing import Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy.core.spidermw import SpiderMiddlewareManager
@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.test import get_crawler
@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase):
self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
def _scrape_response(self):
async def _scrape_response(self) -> Any:
"""Execute spider mw manager's scrape_response method and return the result.
Raise exception in case of failure.
"""
scrape_func = mock.MagicMock()
dfd = self.mwman.scrape_response(
scrape_func, self.response, self.request, self.spider
return await maybe_deferred_to_future(
self.mwman.scrape_response(
scrape_func, self.response, self.request, self.spider
)
)
# catch deferred result and return the value
results = []
dfd.addBoth(results.append)
self._wait(dfd)
return results[0]
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_input method"""
def test_invalid_process_spider_input(self):
@deferred_f_from_coro_f
async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
result = self._scrape_response()
assert isinstance(result, Failure)
assert isinstance(result.value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._scrape_response()
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_output method"""
def test_invalid_process_spider_output(self):
@deferred_f_from_coro_f
async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
result = self._scrape_response()
assert isinstance(result, Failure)
assert isinstance(result.value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._scrape_response()
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_exception method"""
def test_invalid_process_spider_exception(self):
@deferred_f_from_coro_f
async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
return 1
@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
result = self._scrape_response()
assert isinstance(result, Failure)
assert isinstance(result.value, _InvalidOutput)
with pytest.raises(_InvalidOutput):
await self._scrape_response()
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
"""Re raise the exception by returning None"""
def test_process_spider_exception_return_none(self):
@deferred_f_from_coro_f
async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception, spider):
return None
@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
result = self._scrape_response()
assert isinstance(result, Failure)
assert isinstance(result.value, ZeroDivisionError)
with pytest.raises(ZeroDivisionError):
await self._scrape_response()
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):