RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename async test decorators. (#7234)

This commit is contained in:
Andrey Rakhmatullin 2026-02-03 00:39:43 +05:00 committed by GitHub
parent c8aa429c9b
commit 11977afba5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
45 changed files with 513 additions and 515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_addons.py
View File

@ -7,7 +7,7 @@ from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class SimpleAddon:
@ -183,7 +183,7 @@ class TestAddonManager:
extra={"crawler": crawler},
)
@inlineCallbacks
@inline_callbacks_test
def test_enable_addon_in_spider(self):
class MySpider(Spider):
name = "myspider"

16
tests/test_closespider.py
View File

@ -9,7 +9,7 @@ from tests.spiders import (
MaxItemsAndRequestsSpider,
SlowSpider,
)
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
@pytest.mark.requires_http_handler
@ -23,7 +23,7 @@ class TestCloseSpider:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_closespider_itemcount(self):
close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
@ -33,7 +33,7 @@ class TestCloseSpider:
itemcount = crawler.stats.get_value("item_scraped_count")
assert itemcount >= close_on
@inlineCallbacks
@inline_callbacks_test
def test_closespider_pagecount(self):
close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
@ -43,7 +43,7 @@ class TestCloseSpider:
pagecount = crawler.stats.get_value("response_received_count")
assert pagecount >= close_on
@inlineCallbacks
@inline_callbacks_test
def test_closespider_pagecount_no_item(self):
close_on = 5
max_items = 5
@ -63,7 +63,7 @@ class TestCloseSpider:
itemcount = crawler.stats.get_value("item_scraped_count")
assert pagecount <= close_on + itemcount
@inlineCallbacks
@inline_callbacks_test
def test_closespider_pagecount_no_item_with_pagecount(self):
close_on_pagecount_no_item = 5
close_on_pagecount = 20
@ -80,7 +80,7 @@ class TestCloseSpider:
pagecount = crawler.stats.get_value("response_received_count")
assert pagecount < close_on_pagecount
@inlineCallbacks
@inline_callbacks_test
def test_closespider_errorcount(self):
close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
@ -92,7 +92,7 @@ class TestCloseSpider:
assert crawler.stats.get_value("spider_exceptions/count") >= close_on
assert errorcount >= close_on
@inlineCallbacks
@inline_callbacks_test
def test_closespider_timeout(self):
close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
@ -102,7 +102,7 @@ class TestCloseSpider:
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
assert total_seconds >= close_on
@inlineCallbacks
@inline_callbacks_test
def test_closespider_timeout_no_item(self):
timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})

4
tests/test_contracts.py
View File

@ -18,7 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class DemoItem(Item):
@ -502,7 +502,7 @@ class TestContractsManager:
assert self.results.errors
@pytest.mark.requires_http_handler
@inlineCallbacks
@inline_callbacks_test
def test_same_url(self):
class TestSameUrlSpider(Spider):
name = "test_same_url"

14
tests/test_core_downloader.py
View File

@ -25,7 +25,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
@ -97,7 +97,7 @@ class TestContextFactoryBase:
class TestContextFactory(TestContextFactoryBase):
@deferred_f_from_coro_f
@coroutine_test
async def testPayload(self, server_url: str) -> None:
s = "0123456789" * 10
crawler = get_crawler()
@ -135,7 +135,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
)
assert body == to_bytes(s)
@deferred_f_from_coro_f
@coroutine_test
async def test_setting_default(self, server_url: str) -> None:
crawler = get_crawler()
settings = Settings()
@ -155,7 +155,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
with pytest.raises(KeyError):
load_context_factory_from_settings(settings, crawler)
@deferred_f_from_coro_f
@coroutine_test
async def test_setting_explicit(self, server_url: str) -> None:
crawler = get_crawler()
settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})
@ -163,7 +163,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f
@coroutine_test
async def test_direct_from_crawler(self, server_url: str) -> None:
# the setting is ignored
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
@ -171,14 +171,14 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f
@coroutine_test
async def test_direct_init(self, server_url: str) -> None:
client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
def fetch(self, request, spider): # pylint: disable=signature-differs

4
tests/test_core_scraper.py
View File

@ -6,14 +6,14 @@ import pytest
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_scraper_exception(
mockserver: MockServer,
caplog: pytest.LogCaptureFixture,

122
tests/test_crawl.py
View File

@ -55,7 +55,7 @@ from tests.spiders import (
StartGoodAndBadOutput,
StartItemSpider,
)
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
@ -74,17 +74,17 @@ class TestCrawl:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_follow_all(self):
crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert len(crawler.spider.urls_visited) == 11 # 10 + start_url
@deferred_f_from_coro_f
@coroutine_test
async def test_fixed_delay(self):
await self._test_delay(total=3, delay=0.2)
@deferred_f_from_coro_f
@coroutine_test
async def test_randomized_delay(self):
await self._test_delay(total=3, delay=0.1, randomize=True)
@ -122,7 +122,7 @@ class TestCrawl:
average = total_time / (len(times) - 1)
assert average <= delay / tolerance, "test total or delay values are too small"
@inlineCallbacks
@inline_callbacks_test
def test_timeout_success(self):
crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -130,7 +130,7 @@ class TestCrawl:
assert crawler.spider.t2 > 0
assert crawler.spider.t2 > crawler.spider.t1
@inlineCallbacks
@inline_callbacks_test
def test_timeout_failure(self):
crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -147,7 +147,7 @@ class TestCrawl:
assert crawler.spider.t2_err > 0
assert crawler.spider.t2_err > crawler.spider.t1
@inlineCallbacks
@inline_callbacks_test
def test_retry_503(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
@ -156,7 +156,7 @@ class TestCrawl:
)
self._assert_retried(log)
@inlineCallbacks
@inline_callbacks_test
def test_retry_conn_failed(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
@ -165,7 +165,7 @@ class TestCrawl:
)
self._assert_retried(log)
@inlineCallbacks
@inline_callbacks_test
def test_retry_dns_error(self):
if NON_EXISTING_RESOLVABLE:
pytest.skip("Non-existing hosts are resolvable")
@ -177,7 +177,7 @@ class TestCrawl:
)
self._assert_retried(log)
@inlineCallbacks
@inline_callbacks_test
def test_start_bug_before_yield(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider)
@ -188,7 +188,7 @@ class TestCrawl:
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@inlineCallbacks
@inline_callbacks_test
def test_start_bug_yielding(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider)
@ -199,7 +199,7 @@ class TestCrawl:
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@inlineCallbacks
@inline_callbacks_test
def test_start_items(self):
items = []
@ -214,7 +214,7 @@ class TestCrawl:
assert len(log.records) == 0
assert items == [{"name": "test item"}]
@inlineCallbacks
@inline_callbacks_test
def test_start_unsupported_output(self):
"""Anything that is not a request is assumed to be an item, avoiding a
potentially expensive call to itemadapter.is_item(), and letting
@ -235,7 +235,7 @@ class TestCrawl:
assert len(items) == 3
assert not any(isinstance(item, Request) for item in items)
@inlineCallbacks
@inline_callbacks_test
def test_start_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = get_crawler(DuplicateStartSpider, settings)
@ -253,7 +253,7 @@ class TestCrawl:
)
assert crawler.spider.visited == 3
@inlineCallbacks
@inline_callbacks_test
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
@ -285,7 +285,7 @@ with multiples lines
)
assert str(log).count("Got response 200") == 1
@inlineCallbacks
@inline_callbacks_test
def test_retry_conn_lost(self):
# connection lost after receiving data
crawler = get_crawler(SimpleSpider)
@ -295,7 +295,7 @@ with multiples lines
)
self._assert_retried(log)
@inlineCallbacks
@inline_callbacks_test
def test_retry_conn_aborted(self):
# connection lost before receiving data
crawler = get_crawler(SimpleSpider)
@ -309,7 +309,7 @@ with multiples lines
assert str(log).count("Retrying") == 2
assert str(log).count("Gave up retrying") == 1
@inlineCallbacks
@inline_callbacks_test
def test_referer_header(self):
"""Referer header is set by RefererMiddleware unless it is already set"""
req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1)
@ -337,7 +337,7 @@ with multiples lines
echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body))
assert echo3["headers"].get("Referer") == ["http://example.com"]
@inlineCallbacks
@inline_callbacks_test
def test_engine_status(self):
est = []
@ -353,7 +353,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == 1
@inlineCallbacks
@inline_callbacks_test
def test_format_engine_status(self):
est = []
@ -376,7 +376,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"
@inlineCallbacks
@inline_callbacks_test
def test_open_spider_error_on_faulty_pipeline(self):
settings = {
"ITEM_PIPELINES": {
@ -390,7 +390,7 @@ with multiples lines
)
assert not crawler.crawling
@inlineCallbacks
@inline_callbacks_test
def test_crawlerrunner_accepts_crawler(self):
crawler = get_crawler(SimpleSpider)
runner = CrawlerRunner()
@ -402,7 +402,7 @@ with multiples lines
)
assert "Got response 200" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):
runner = CrawlerRunner(get_reactor_settings())
runner.crawl(
@ -422,7 +422,7 @@ with multiples lines
self._assert_retried(caplog.text)
assert "Got response 200" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(SimpleSpider)
await maybe_deferred_to_future(crawler.crawl("foo://bar"))
@ -461,7 +461,7 @@ class TestCrawlSpider:
assert crawler.stats
return log, items, crawler.stats
@inlineCallbacks
@inline_callbacks_test
def test_crawlspider_with_parse(self):
crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log:
@ -471,7 +471,7 @@ class TestCrawlSpider:
assert "[parse] status 201 (foo: None)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_crawlspider_with_async_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncCallback)
with LogCapture() as log:
@ -481,7 +481,7 @@ class TestCrawlSpider:
assert "[parse_async] status 201 (foo: None)" in str(log)
assert "[parse_async] status 202 (foo: bar)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_crawlspider_with_async_generator_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
with LogCapture() as log:
@ -491,7 +491,7 @@ class TestCrawlSpider:
assert "[parse_async_gen] status 201 (foo: None)" in str(log)
assert "[parse_async_gen] status 202 (foo: bar)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_crawlspider_with_errback(self):
crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log:
@ -504,7 +504,7 @@ class TestCrawlSpider:
assert "[errback] status 500" in str(log)
assert "[errback] status 501" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_crawlspider_process_request_cb_kwargs(self):
crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
with LogCapture() as log:
@ -514,7 +514,7 @@ class TestCrawlSpider:
assert "[parse] status 201 (foo: process_request)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_async_def_parse(self):
crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log:
@ -524,7 +524,7 @@ class TestCrawlSpider:
assert "Got response 200" in str(log)
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_async_def_asyncio_parse(self):
crawler = get_crawler(
AsyncDefAsyncioSpider,
@ -539,7 +539,7 @@ class TestCrawlSpider:
assert "Got response 200" in str(log)
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncio_parse_items_list(self):
log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider)
assert "Got response 200" in str(log)
@ -547,7 +547,7 @@ class TestCrawlSpider:
assert {"id": 2} in items
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_async_def_asyncio_parse_items_single_element(self):
items = []
@ -564,7 +564,7 @@ class TestCrawlSpider:
assert {"foo": 42} in items
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncgen_parse(self):
log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider)
assert "Got response 200" in str(log)
@ -572,7 +572,7 @@ class TestCrawlSpider:
assert itemcount == 1
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncgen_parse_loop(self):
log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider)
assert "Got response 200" in str(log)
@ -582,7 +582,7 @@ class TestCrawlSpider:
assert {"foo": i} in items
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncgen_parse_exc(self):
log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider)
log = str(log)
@ -594,7 +594,7 @@ class TestCrawlSpider:
assert {"foo": i} in items
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncgen_parse_complex(self):
_, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider)
itemcount = stats.get_value("item_scraped_count")
@ -606,30 +606,30 @@ class TestCrawlSpider:
assert {"index2": i} in items
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_asyncio_parse_reqs_list(self):
log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider)
for req_id in range(3):
assert f"Got response 200, req_id {req_id}" in str(log)
@pytest.mark.only_not_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_deferred_direct(self):
_, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider)
assert items == [{"code": 200}]
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_deferred_wrapped(self):
_, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
assert items == [{"code": 200}]
@deferred_f_from_coro_f
@coroutine_test
async def test_async_def_deferred_maybe_wrapped(self):
_, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
assert items == [{"code": 200}]
@inlineCallbacks
@inline_callbacks_test
def test_response_ssl_certificate_none(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False)
@ -649,7 +649,7 @@ class TestCrawlSpider:
),
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_response_ssl_certificate(
self, mockserver: MockServer, url: str
) -> None:
@ -675,7 +675,7 @@ class TestCrawlSpider:
),
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None:
crawler = get_crawler(SingleRequestSpider)
url = mockserver.url(url)
@ -686,7 +686,7 @@ class TestCrawlSpider:
assert isinstance(ip_address, IPv4Address)
assert str(ip_address) == gethostbyname(expected_netloc)
@inlineCallbacks
@inline_callbacks_test
def test_bytes_received_stop_download_callback(self):
crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -700,7 +700,7 @@ class TestCrawlSpider:
< crawler.spider.full_response_length
)
@inlineCallbacks
@inline_callbacks_test
def test_bytes_received_stop_download_errback(self):
crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -716,7 +716,7 @@ class TestCrawlSpider:
< crawler.spider.full_response_length
)
@inlineCallbacks
@inline_callbacks_test
def test_headers_received_stop_download_callback(self):
crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -726,7 +726,7 @@ class TestCrawlSpider:
"headers_received"
)
@inlineCallbacks
@inline_callbacks_test
def test_headers_received_stop_download_errback(self):
crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -738,7 +738,7 @@ class TestCrawlSpider:
"failure"
].value.response.headers == crawler.spider.meta.get("headers_received")
@inlineCallbacks
@inline_callbacks_test
def test_spider_callback_deferred_deprecated(self):
def cb(response: Response) -> Any:
return succeed(None)
@ -750,7 +750,7 @@ class TestCrawlSpider:
):
yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback(self):
failures = []
@ -767,7 +767,7 @@ class TestCrawlSpider:
assert "HTTP status code is not handled or not allowed" in str(log)
assert "Spider error processing" not in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_silence(self):
failures = []
@ -783,7 +783,7 @@ class TestCrawlSpider:
assert "HTTP status code is not handled or not allowed" not in str(log)
assert "Spider error processing" not in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_exception(self):
def eb(failure: Failure) -> None:
raise ValueError("foo")
@ -795,7 +795,7 @@ class TestCrawlSpider:
)
assert "Spider error processing" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_item(self):
def eb(failure: Failure) -> Any:
return {"foo": "bar"}
@ -809,7 +809,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_request(self):
def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/"))
@ -823,7 +823,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_downloader_error(self):
failures = []
@ -840,7 +840,7 @@ class TestCrawlSpider:
assert "Error downloading" in str(log)
assert "Spider error processing" not in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_downloader_error_exception(self):
def eb(failure: Failure) -> None:
raise ValueError("foo")
@ -853,7 +853,7 @@ class TestCrawlSpider:
assert "Error downloading" in str(log)
assert "Spider error processing" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_downloader_error_item(self):
def eb(failure: Failure) -> Any:
return {"foo": "bar"}
@ -867,7 +867,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_downloader_error_request(self):
def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/"))
@ -881,7 +881,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_spider_errback_deferred_deprecated(self):
def eb(failure: Failure) -> Any:
return succeed(None)
@ -895,7 +895,7 @@ class TestCrawlSpider:
seed=self.mockserver.url("/status?n=400"), errback_func=eb
)
@inlineCallbacks
@inline_callbacks_test
def test_raise_closespider(self):
def cb(response):
raise CloseSpider
@ -906,7 +906,7 @@ class TestCrawlSpider:
assert "Closing spider (cancelled)" in str(log)
assert "Spider error processing" not in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_raise_closespider_reason(self):
def cb(response):
raise CloseSpider("my_reason")

42
tests/test_crawler.py
View File

@ -41,7 +41,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.mockserver.http import MockServer
from tests.utils import get_script_run_env
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
BASE_SETTINGS: dict[str, Any] = {}
@ -101,21 +101,21 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(ValueError, match="spidercls argument must be a class"):
Crawler(DefaultSpider())
@inlineCallbacks
@inline_callbacks_test
def test_crawler_crawl_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with pytest.raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler_crawl_async_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
await crawler.crawl_async()
with pytest.raises(RuntimeError, match="more than once on the same instance"):
await crawler.crawl_async()
@inlineCallbacks
@inline_callbacks_test
def test_crawler_crawl_twice_parallel_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
d1 = crawler.crawl()
@ -125,7 +125,7 @@ class TestCrawler(TestBaseCrawler):
yield d2
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler_crawl_async_twice_parallel_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
t1 = asyncio.create_task(crawler.crawl_async())
@ -172,7 +172,7 @@ class TestCrawler(TestBaseCrawler):
addon = crawler.get_addon(ChildAddon)
assert addon is None
@inlineCallbacks
@inline_callbacks_test
def test_get_downloader_middleware(self):
class ParentDownloaderMiddleware:
pass
@ -235,7 +235,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
crawler.get_downloader_middleware(DefaultSpider)
@inlineCallbacks
@inline_callbacks_test
def test_get_downloader_middleware_no_engine(self):
class MySpider(Spider):
name = "myspider"
@ -252,7 +252,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@inline_callbacks_test
def test_get_extension(self):
class ParentExtension:
pass
@ -315,7 +315,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
crawler.get_extension(DefaultSpider)
@inlineCallbacks
@inline_callbacks_test
def test_get_extension_no_engine(self):
class MySpider(Spider):
name = "myspider"
@ -332,7 +332,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@inline_callbacks_test
def test_get_item_pipeline(self):
class ParentItemPipeline:
pass
@ -395,7 +395,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
crawler.get_item_pipeline(DefaultSpider)
@inlineCallbacks
@inline_callbacks_test
def test_get_item_pipeline_no_engine(self):
class MySpider(Spider):
name = "myspider"
@ -412,7 +412,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@inline_callbacks_test
def test_get_spider_middleware(self):
class ParentSpiderMiddleware:
pass
@ -475,7 +475,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError):
crawler.get_spider_middleware(DefaultSpider)
@inlineCallbacks
@inline_callbacks_test
def test_get_spider_middleware_no_engine(self):
class MySpider(Spider):
name = "myspider"
@ -516,7 +516,7 @@ class TestCrawlerLogging:
get_crawler(MySpider)
assert get_scrapy_root_handler() is None
@deferred_f_from_coro_f
@coroutine_test
async def test_spider_custom_settings_log_level(self, tmp_path):
log_file = Path(tmp_path, "log.txt")
log_file.write_text("previous message\n", encoding="utf-8")
@ -686,20 +686,20 @@ class TestCrawlerRunnerHasSpider:
def _crawl(runner, spider):
return runner.crawl(spider)
@inlineCallbacks
@inline_callbacks_test
def test_crawler_runner_bootstrap_successful(self):
runner = self._runner()
yield self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@inlineCallbacks
@inline_callbacks_test
def test_crawler_runner_bootstrap_successful_for_several(self):
runner = self._runner()
yield self._crawl(runner, NoRequestsSpider)
yield self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@inlineCallbacks
@inline_callbacks_test
def test_crawler_runner_bootstrap_failed(self):
runner = self._runner()
@ -712,7 +712,7 @@ class TestCrawlerRunnerHasSpider:
assert runner.bootstrap_failed
@inlineCallbacks
@inline_callbacks_test
def test_crawler_runner_bootstrap_failed_for_several(self):
runner = self._runner()
@ -727,7 +727,7 @@ class TestCrawlerRunnerHasSpider:
assert runner.bootstrap_failed
@inlineCallbacks
@inline_callbacks_test
def test_crawler_runner_asyncio_enabled_true(
self, reactor_pytest: str
) -> Generator[Deferred[Any], Any, None]:
@ -959,7 +959,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.expect_exact("Spider closed (shutdown)")
p.wait()
@inlineCallbacks
@inline_callbacks_test
def test_shutdown_forced(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "10")
@ -1297,7 +1297,7 @@ def test_log_scrapy_info(settings, items, caplog):
assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_crawler_stop() -> None:
crawler = get_crawler(DefaultSpider)
d = crawler.crawl()

34
tests/test_downloader_handlers.py
View File

@ -21,7 +21,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class DummyDH:
@ -126,7 +126,7 @@ class TestFile:
os.close(self.fd)
Path(self.tmpname).unlink()
@deferred_f_from_coro_f
@coroutine_test
async def test_download(self):
request = Request(path_to_file_uri(self.tmpname))
assert request.url.upper().endswith("%5E")
@ -136,7 +136,7 @@ class TestFile:
assert response.body == b"0123456789"
assert response.protocol is None
@deferred_f_from_coro_f
@coroutine_test
async def test_non_existent(self):
request = Request(path_to_file_uri(mkdtemp()))
# the specific exception differs between platforms
@ -163,7 +163,7 @@ class TestS3Anon:
self.s3reqh = build_from_crawler(S3DownloadHandler, crawler)
self.download_request = self.s3reqh.download_request
@deferred_f_from_coro_f
@coroutine_test
async def test_anon_request(self):
req = Request("s3://aws-publicdatasets/")
httpreq = await self.download_request(req)
@ -205,7 +205,7 @@ class TestS3:
mock_formatdate.return_value = date
yield
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing1(self):
# gets an object from the johnsmith bucket.
date = "Tue, 27 Mar 2007 19:36:42 +0000"
@ -217,7 +217,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing2(self):
# puts an object into the johnsmith bucket.
date = "Tue, 27 Mar 2007 21:15:45 +0000"
@ -237,7 +237,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ="
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing3(self):
# lists the content of the johnsmith bucket.
date = "Tue, 27 Mar 2007 19:42:41 +0000"
@ -256,7 +256,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4="
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
date = "Tue, 27 Mar 2007 19:44:46 +0000"
@ -268,7 +268,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g="
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.
date = "Tue, 27 Mar 2007 21:06:08 +0000"
@ -297,7 +297,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI="
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_signing7(self):
# ensure that spaces are quoted properly before signing
date = "Tue, 27 Mar 2007 19:42:41 +0000"
@ -327,7 +327,7 @@ class TestDataURI:
download_handler = build_from_crawler(DataURIDownloadHandler, crawler)
self.download_request = download_handler.download_request
@deferred_f_from_coro_f
@coroutine_test
async def test_response_attrs(self):
uri = "data:,A%20brief%20note"
request = Request(uri)
@ -335,7 +335,7 @@ class TestDataURI:
assert response.url == uri
assert not response.headers
@deferred_f_from_coro_f
@coroutine_test
async def test_default_mediatype_encoding(self):
request = Request("data:,A%20brief%20note")
response = await self.download_request(request)
@ -343,7 +343,7 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "US-ASCII"
@deferred_f_from_coro_f
@coroutine_test
async def test_default_mediatype(self):
request = Request("data:;charset=iso-8859-7,%be%d3%be")
response = await self.download_request(request)
@ -351,7 +351,7 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "iso-8859-7"
@deferred_f_from_coro_f
@coroutine_test
async def test_text_charset(self):
request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
response = await self.download_request(request)
@ -359,7 +359,7 @@ class TestDataURI:
assert response.body == b"\xbe\xd3\xbe"
assert response.encoding == "iso-8859-7"
@deferred_f_from_coro_f
@coroutine_test
async def test_mediatype_parameters(self):
request = Request(
"data:text/plain;foo=%22foo;bar%5C%22%22;"
@ -371,13 +371,13 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "utf-8"
@deferred_f_from_coro_f
@coroutine_test
async def test_base64(self):
request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")
response = await self.download_request(request)
assert response.text == "Hello, world."
@deferred_f_from_coro_f
@coroutine_test
async def test_protocol(self):
request = Request("data:,")
response = await self.download_request(request)

100
tests/test_downloader_handlers_http_base.py
View File

@ -34,7 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import SingleRequestSpider
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
@ -63,21 +63,21 @@ class TestHttpBase(ABC):
finally:
await dh.close()
@deferred_f_from_coro_f
@coroutine_test
async def test_unsupported_scheme(self) -> None:
request = Request("ftp://unsupported.scheme")
async with self.get_dh() as download_handler:
with pytest.raises(UnsupportedURLSchemeError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"Works"
@deferred_f_from_coro_f
@coroutine_test
async def test_download_head(self, mockserver: MockServer) -> None:
request = Request(
mockserver.url("/text", is_secure=self.is_secure), method="HEAD"
@ -94,7 +94,7 @@ class TestHttpBase(ABC):
if http_status.value == 200 or http_status.value // 100 in (4, 5)
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_has_correct_http_status_code(
self, mockserver: MockServer, http_status: HTTPStatus
) -> None:
@ -105,7 +105,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert response.status == http_status.value
@deferred_f_from_coro_f
@coroutine_test
async def test_server_receives_correct_request_headers(
self, mockserver: MockServer
) -> None:
@ -132,7 +132,7 @@ class TestHttpBase(ABC):
assert header_name in body["headers"]
assert body["headers"][header_name] == [header_value]
@deferred_f_from_coro_f
@coroutine_test
async def test_request_header_none(self, mockserver: MockServer) -> None:
"""Adding a header with None as the value should not send that header."""
request_headers = {
@ -158,7 +158,7 @@ class TestHttpBase(ABC):
[("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")],
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_request_header_duplicate(
self, mockserver: MockServer, request_headers: Any
) -> None:
@ -174,7 +174,7 @@ class TestHttpBase(ABC):
assert "headers" in body
assert body["headers"]["X-Custom-Header"] == ["foo", "bar"]
@deferred_f_from_coro_f
@coroutine_test
async def test_server_receives_correct_request_body(
self, mockserver: MockServer
) -> None:
@ -191,7 +191,7 @@ class TestHttpBase(ABC):
body = json.loads(response.body.decode("utf-8"))
assert json.loads(body["body"]) == request_body
@deferred_f_from_coro_f
@coroutine_test
async def test_download_has_correct_response_headers(
self, mockserver: MockServer
) -> None:
@ -230,14 +230,14 @@ class TestHttpBase(ABC):
header_value, encoding="utf-8"
)
@deferred_f_from_coro_f
@coroutine_test
async def test_redirect_status(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/redirect", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.status == 302
@deferred_f_from_coro_f
@coroutine_test
async def test_redirect_status_head(self, mockserver: MockServer) -> None:
request = Request(
mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD"
@ -246,7 +246,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert response.status == 302
@deferred_f_from_coro_f
@coroutine_test
async def test_timeout_download_from_spider_nodata_rcvd(
self, mockserver: MockServer, reactor_pytest: str
) -> None:
@ -264,7 +264,7 @@ class TestHttpBase(ABC):
with pytest.raises(DownloadTimeoutError):
await maybe_deferred_to_future(d)
@deferred_f_from_coro_f
@coroutine_test
async def test_timeout_download_from_spider_server_hangs(
self,
mockserver: MockServer,
@ -286,7 +286,7 @@ class TestHttpBase(ABC):
await maybe_deferred_to_future(d)
@pytest.mark.parametrize("send_header", [True, False])
@deferred_f_from_coro_f
@coroutine_test
async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None:
host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}"
request = Request(
@ -301,7 +301,7 @@ class TestHttpBase(ABC):
else:
assert not request.headers
@deferred_f_from_coro_f
@coroutine_test
async def test_content_length_zero_bodyless_post_request_headers(
self, mockserver: MockServer
) -> None:
@ -322,7 +322,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert response.body == b"0"
@deferred_f_from_coro_f
@coroutine_test
async def test_content_length_zero_bodyless_post_only_one(
self, mockserver: MockServer
) -> None:
@ -336,7 +336,7 @@ class TestHttpBase(ABC):
assert len(contentlengths) == 1
assert contentlengths == [b"0"]
@deferred_f_from_coro_f
@coroutine_test
async def test_payload(self, mockserver: MockServer) -> None:
body = b"1" * 100 # PayloadResource requires body length to be 100
request = Request(
@ -348,7 +348,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert response.body == body
@deferred_f_from_coro_f
@coroutine_test
async def test_response_header_content_length(self, mockserver: MockServer) -> None:
request = Request(
mockserver.url("/text", is_secure=self.is_secure), method="GET"
@ -364,7 +364,7 @@ class TestHttpBase(ABC):
("foo", b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_response_class(
self,
filename: str,
@ -379,14 +379,14 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert type(response) is response_class # pylint: disable=unidiomatic-typecheck
@deferred_f_from_coro_f
@coroutine_test
async def test_get_duplicate_header(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
@deferred_f_from_coro_f
@coroutine_test
async def test_download_is_not_automatically_gzip_decoded(
self, mockserver: MockServer
) -> None:
@ -420,7 +420,7 @@ class TestHttpBase(ABC):
expected_decoding = bytes(data, encoding="utf-8")
assert gzip.decompress(response.body) == expected_decoding
@deferred_f_from_coro_f
@coroutine_test
async def test_no_cookie_processing_or_persistence(
self, mockserver: MockServer
) -> None:
@ -451,14 +451,14 @@ class TestHttpBase(ABC):
class TestHttp11Base(TestHttpBase):
"""HTTP 1.1 test case"""
@deferred_f_from_coro_f
@coroutine_test
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"Works"
@deferred_f_from_coro_f
@coroutine_test
async def test_response_class_choosing_request(
self, mockserver: MockServer
) -> None:
@ -473,7 +473,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request)
assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_maxsize(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure))
@ -487,7 +487,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_maxsize_very_large_file(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None:
@ -498,7 +498,7 @@ class TestHttp11Base(TestHttpBase):
assert "larger than download max size" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None:
meta = {"download_maxsize": 2}
request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta)
@ -506,7 +506,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_small_maxsize_via_setting(
self, mockserver: MockServer
) -> None:
@ -515,7 +515,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_large_maxsize_via_setting(
self, mockserver: MockServer
) -> None:
@ -524,7 +524,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request)
assert response.body == b"Works"
@deferred_f_from_coro_f
@coroutine_test
async def test_download_chunked_content(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/chunked", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
@ -532,7 +532,7 @@ class TestHttp11Base(TestHttpBase):
assert response.body == b"chunked content\n"
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f
@coroutine_test
async def test_download_cause_data_loss(
self, url: str, mockserver: MockServer
) -> None:
@ -541,7 +541,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(ResponseDataLossError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_cause_data_loss_double_warning(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
@ -557,7 +557,7 @@ class TestHttp11Base(TestHttpBase):
assert "Got data loss" not in caplog.text
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f
@coroutine_test
async def test_download_allow_data_loss(
self, url: str, mockserver: MockServer
) -> None:
@ -570,7 +570,7 @@ class TestHttp11Base(TestHttpBase):
assert response.flags == ["dataloss"]
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f
@coroutine_test
async def test_download_allow_data_loss_via_setting(
self, url: str, mockserver: MockServer
) -> None:
@ -581,7 +581,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request)
assert response.flags == ["dataloss"]
@deferred_f_from_coro_f
@coroutine_test
async def test_download_conn_failed(self) -> None:
# copy of TestCrawl.test_retry_conn_failed()
scheme = "https" if self.is_secure else "http"
@ -590,7 +590,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadConnectionRefusedError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_conn_lost(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_retry_conn_lost()
request = Request(mockserver.url("/drop?abort=0", is_secure=self.is_secure))
@ -598,7 +598,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(ResponseDataLossError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_conn_aborted(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_retry_conn_aborted()
request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure))
@ -609,7 +609,7 @@ class TestHttp11Base(TestHttpBase):
@pytest.mark.skipif(
NON_EXISTING_RESOLVABLE, reason="Non-existing hosts are resolvable"
)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_dns_error(self) -> None:
# copy of TestCrawl.test_retry_dns_error()
scheme = "https" if self.is_secure else "http"
@ -618,7 +618,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(CannotResolveHostError):
await download_handler.download_request(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_protocol(self, mockserver: MockServer) -> None:
request = Request(
mockserver.url("/host", is_secure=self.is_secure), method="GET"
@ -642,7 +642,7 @@ class TestHttps11Base(TestHttp11Base):
# (not just Scrapy) hang on /drop?abort=0.
pytest.skip("Unable to test on HTTPS")
@deferred_f_from_coro_f
@coroutine_test
async def test_tls_logging(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None:
@ -695,7 +695,7 @@ class TestSimpleHttpsBase(ABC):
finally:
await dh.close()
@deferred_f_from_coro_f
@coroutine_test
async def test_download(self, url: str) -> None:
request = Request(url)
async with self.get_dh() as download_handler:
@ -738,7 +738,7 @@ class TestHttpWithCrawlerBase(ABC):
is_secure = False
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_content_length(self, mockserver: MockServer) -> None:
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
# http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
@ -755,7 +755,7 @@ class TestHttpWithCrawlerBase(ABC):
failure = crawler.spider.meta["failure"] # type: ignore[attr-defined]
assert isinstance(failure.value, DownloadCancelledError)
@deferred_f_from_coro_f
@coroutine_test
async def test_download(self, mockserver: MockServer) -> None:
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
await maybe_deferred_to_future(
@ -769,7 +769,7 @@ class TestHttpWithCrawlerBase(ABC):
reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]
assert reason == "finished"
@deferred_f_from_coro_f
@coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
if not self.is_secure:
pytest.skip("Only applies to HTTPS")
@ -784,7 +784,7 @@ class TestHttpWithCrawlerBase(ABC):
assert cert.getSubject().commonName == b"localhost"
assert cert.getIssuer().commonName == b"localhost"
@deferred_f_from_coro_f
@coroutine_test
async def test_response_ip_address(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_response_ip_address()
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
@ -821,7 +821,7 @@ class TestHttpProxyBase(ABC):
finally:
await dh.close()
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_proxy(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:
@ -833,7 +833,7 @@ class TestHttpProxyBase(ABC):
assert response.url == request.url
assert response.body == self.expected_http_proxy_request_body
@deferred_f_from_coro_f
@coroutine_test
async def test_download_without_proxy(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:
@ -846,7 +846,7 @@ class TestHttpProxyBase(ABC):
assert response.url == request.url
assert response.body == b"/path/to/resource"
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_proxy_https_timeout(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:
@ -860,7 +860,7 @@ class TestHttpProxyBase(ABC):
await download_handler.download_request(request)
assert domain in str(exc_info.value)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_with_proxy_without_http_scheme(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:

28
tests/test_downloadermiddleware.py
View File

@ -16,7 +16,7 @@ from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@ -60,7 +60,7 @@ class TestManagerBase:
class TestDefaults(TestManagerBase):
"""Tests default behavior with default settings"""
@deferred_f_from_coro_f
@coroutine_test
async def test_request_response(self):
req = Request("http://example.com/index.html")
resp = Response(req.url, status=200)
@ -68,7 +68,7 @@ class TestDefaults(TestManagerBase):
ret = await self._download(mwman, req, resp)
assert isinstance(ret, Response), "Non-response returned"
@deferred_f_from_coro_f
@coroutine_test
async def test_3xx_and_invalid_gzipped_body_must_redirect(self):
"""Regression test for a failure when redirecting a compressed
request.
@ -101,7 +101,7 @@ class TestDefaults(TestManagerBase):
"Not redirected to location header"
)
@deferred_f_from_coro_f
@coroutine_test
async def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com")
body = b"<p>You are being redirected</p>"
@ -124,7 +124,7 @@ class TestDefaults(TestManagerBase):
class TestResponseFromProcessRequest(TestManagerBase):
"""Tests middleware returning a response from process_request."""
@deferred_f_from_coro_f
@coroutine_test
async def test_download_func_not_called(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
@ -144,7 +144,7 @@ class TestResponseFromProcessRequest(TestManagerBase):
class TestResponseFromProcessException(TestManagerBase):
"""Tests middleware returning a response from process_exception."""
@deferred_f_from_coro_f
@coroutine_test
async def test_process_response_called(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
@ -173,7 +173,7 @@ class TestResponseFromProcessException(TestManagerBase):
class TestInvalidOutput(TestManagerBase):
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_request(self):
"""Invalid return value for process_request method should raise an exception"""
req = Request("http://example.com/index.html")
@ -187,7 +187,7 @@ class TestInvalidOutput(TestManagerBase):
with pytest.raises(_InvalidOutput):
await self._download(mwman, req)
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_response(self):
"""Invalid return value for process_response method should raise an exception"""
req = Request("http://example.com/index.html")
@ -201,7 +201,7 @@ class TestInvalidOutput(TestManagerBase):
with pytest.raises(_InvalidOutput):
await self._download(mwman, req)
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_exception(self):
"""Invalid return value for process_exception method should raise an exception"""
req = Request("http://example.com/index.html")
@ -222,7 +222,7 @@ class TestInvalidOutput(TestManagerBase):
class TestMiddlewareUsingDeferreds(TestManagerBase):
"""Middlewares using Deferreds (deprecated) should work"""
@deferred_f_from_coro_f
@coroutine_test
async def test_deferred(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
@ -252,7 +252,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
class TestMiddlewareUsingCoro(TestManagerBase):
"""Middlewares using asyncio coroutines should work"""
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncdef(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
assert not download_func.called
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncdef_asyncio(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
@ -289,7 +289,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
class TestDownloadDeprecated(TestManagerBase):
@deferred_f_from_coro_f
@coroutine_test
async def test_mwman_download(self):
req = Request("http://example.com/index.html")
resp = Response(req.url, status=200)
@ -309,7 +309,7 @@ class TestDownloadDeprecated(TestManagerBase):
class TestDeprecatedSpiderArg(TestManagerBase):
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")

24
tests/test_downloadermiddleware_robotstxt.py
View File

@ -16,7 +16,7 @@ from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.test_robotstxt_interface import rerp_available
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@ -60,7 +60,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response
return crawler
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@ -74,7 +74,7 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware
)
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
d1 = deferred_from_coro(
@ -86,20 +86,20 @@ Disallow: /some/randome/page.html
await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True))
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2)
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_meta(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
meta = {"dont_obey_robotstxt": True}
@ -128,7 +128,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response
return crawler
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
@ -150,7 +150,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response
return crawler
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_empty_response(self):
# empty response should equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
@ -158,7 +158,7 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None:
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found")
@ -174,7 +174,7 @@ Disallow: /some/randome/page.html
await middleware.process_request(Request("http://site.local"))
assert "Robotstxt address not found" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_immediate_error(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found")
@ -187,7 +187,7 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler)
await self.assertNotIgnored(Request("http://site.local"), middleware)
@deferred_f_from_coro_f
@coroutine_test
async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
@ -216,7 +216,7 @@ Disallow: /some/randome/page.html
middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@deferred_f_from_coro_f
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock()

6
tests/test_downloaderslotssettings.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MetaSpider
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
class DownloaderSlotsSettingsTestSpider(MetaSpider):
@ -68,7 +68,7 @@ class TestCrawl:
self.runner = CrawlerRunner()
@pytest.mark.requires_http_handler
@inlineCallbacks
@inline_callbacks_test
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -137,7 +137,7 @@ def test_get_slot_deprecated_spider_arg():
"scrapy.pqueues.DownloaderAwarePriorityQueue",
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_none_slot_with_priority_queue(
mockserver: MockServer, priority_queue_class: str
) -> None:

46
tests/test_engine.py
View File

@ -35,7 +35,7 @@ from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import get_testdata
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from scrapy.core.scheduler import Scheduler
@ -374,7 +374,7 @@ class TestEngineBase:
class TestEngine(TestEngineBase):
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler(self, mockserver: MockServer) -> None:
for spider in (
MySpider,
@ -392,7 +392,7 @@ class TestEngine(TestEngineBase):
self._assert_bytes_received(run)
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
run = CrawlerRun(DupeFilterSpider)
await run.run(mockserver)
@ -400,14 +400,14 @@ class TestEngine(TestEngineBase):
self._assert_dropped_requests(run)
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
run = CrawlerRun(ItemZeroDivisionErrorSpider)
await run.run(mockserver)
self._assert_items_error(run)
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler_change_close_reason_on_idle(
self, mockserver: MockServer
) -> None:
@ -418,7 +418,7 @@ class TestEngine(TestEngineBase):
"reason": "custom_reason",
} == run.signals_caught[signals.spider_closed]
@deferred_f_from_coro_f
@coroutine_test
async def test_close_downloader(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
await e.close_async()
@ -436,7 +436,7 @@ class TestEngine(TestEngineBase):
get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None
)
@inlineCallbacks
@inline_callbacks_test
def test_start_already_running_exception(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
@ -449,7 +449,7 @@ class TestEngine(TestEngineBase):
yield deferred_from_coro(e.stop_async())
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_start_already_running_exception_asyncio(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
@ -460,7 +460,7 @@ class TestEngine(TestEngineBase):
await asyncio.gather(e.start_async(), e.start_async())
await e.stop_async()
@inlineCallbacks
@inline_callbacks_test
def test_start_request_processing_exception(self):
class BadRequestFingerprinter:
def fingerprint(self, request):
@ -527,7 +527,7 @@ class TestEngineDownloadAsync:
async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await engine.download_async(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_async_success(self, engine):
"""Test basic successful async download of a request."""
request = Request("http://example.com")
@ -543,7 +543,7 @@ class TestEngineDownloadAsync:
engine._slot.remove_request.assert_called_once_with(request)
engine.downloader.fetch.assert_called_once_with(request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_async_redirect(self, engine):
"""Test async download with a redirect request."""
original_request = Request("http://example.com")
@ -569,7 +569,7 @@ class TestEngineDownloadAsync:
[call(original_request), call(redirect_request)]
)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_async_no_spider(self, engine):
"""Test async download attempt when no spider is available."""
request = Request("http://example.com")
@ -577,7 +577,7 @@ class TestEngineDownloadAsync:
with pytest.raises(RuntimeError, match="No open spider to crawl:"):
await self._download(engine, request)
@deferred_f_from_coro_f
@coroutine_test
async def test_download_async_failure(self, engine):
"""Test async download when the downloader raises an exception."""
request = Request("http://example.com")
@ -647,7 +647,7 @@ class TestEngineCloseSpider:
crawler.spider = crawler._create_spider()
return crawler
@deferred_f_from_coro_f
@coroutine_test
async def test_no_slot(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
@ -660,14 +660,14 @@ class TestEngineCloseSpider:
engine._slot = slot
await engine.close_spider_async()
@deferred_f_from_coro_f
@coroutine_test
async def test_no_spider(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.close_spider_async()
engine.downloader.close() # cleanup
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_slot(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -679,7 +679,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Slot close failure" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_downloader(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -690,7 +690,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Downloader close failure" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_scraper(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -701,7 +701,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Scraper close failure" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_scheduler(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -713,7 +713,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Scheduler close failure" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_signal(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -731,7 +731,7 @@ class TestEngineCloseSpider:
reason="cancelled",
)
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_stats(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -742,7 +742,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Stats close failure" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
@ -752,7 +752,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text
@deferred_f_from_coro_f
@coroutine_test
async def test_exception_async_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:

14
tests/test_engine_loop.py
View File

@ -12,7 +12,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.test_scheduler import MemoryScheduler
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.http import Response
@ -28,7 +28,7 @@ async def sleep(seconds: float = 0.001) -> None:
class TestMain:
@pytest.mark.requires_reactor # TODO
@deferred_f_from_coro_f
@coroutine_test
async def test_sleep(self):
"""Neither asynchronous sleeps on Spider.start() nor the equivalent on
the scheduler (returning no requests while also returning True from
@ -87,7 +87,7 @@ class TestMain:
expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@deferred_f_from_coro_f
@coroutine_test
async def test_close_during_start_iteration(
self, caplog: pytest.LogCaptureFixture
) -> None:
@ -188,7 +188,7 @@ class TestRequestSendOrder:
expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
@deferred_f_from_coro_f
@coroutine_test
async def test_default(self):
"""By default, callback requests take priority over start requests and
are sent in order. Priority matters, but given the same priority, a
@ -228,7 +228,7 @@ class TestRequestSendOrder:
parse_fn=parse,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_lifo_start(self):
"""Changing the queues of start requests to LIFO, matching the queues
of non-start requests, does not cause all requests to be stored in the
@ -271,7 +271,7 @@ class TestRequestSendOrder:
parse_fn=parse,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_shared_queues(self):
"""If SCHEDULER_START_*_QUEUE is falsy, start requests and other
requests share the same queue, i.e. start requests are not priorized
@ -333,7 +333,7 @@ class TestRequestSendOrder:
# spiders.
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_lazy(self):
start_nums = [1, 2, 4]
cb_nums = [3]

4
tests/test_engine_stop_download_bytes.py
View File

@ -13,7 +13,7 @@ from tests.test_engine import (
MySpider,
TestEngineBase,
)
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
@ -27,7 +27,7 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase):
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None:

4
tests/test_engine_stop_download_headers.py
View File

@ -13,7 +13,7 @@ from tests.test_engine import (
MySpider,
TestEngineBase,
)
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
@ -27,7 +27,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase):
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None:

8
tests/test_extension_telnet.py
View File

@ -4,7 +4,7 @@ from twisted.cred import credentials
from scrapy.extensions.telnet import TelnetConsole
from scrapy.utils.test import get_crawler
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
pytestmark = pytest.mark.requires_reactor
@ -23,7 +23,7 @@ class TestTelnetExtension:
return console, portal
@inlineCallbacks
@inline_callbacks_test
def test_bad_credentials(self):
console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword(b"username", b"password")
@ -32,7 +32,7 @@ class TestTelnetExtension:
yield d
console.stop_listening()
@inlineCallbacks
@inline_callbacks_test
def test_good_credentials(self):
console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword(
@ -42,7 +42,7 @@ class TestTelnetExtension:
yield d
console.stop_listening()
@inlineCallbacks
@inline_callbacks_test
def test_custom_credentials(self):
settings = {
"TELNETCONSOLE_USERNAME": "user",

136
tests/test_feedexport.py
View File

@ -55,7 +55,7 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@ -192,7 +192,7 @@ class TestFTPFeedStorage:
finally:
path.unlink()
@deferred_f_from_coro_f
@coroutine_test
async def test_append(self):
with MockFTPServer() as ftp_server:
filename = "file"
@ -202,7 +202,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", feed_options=feed_options)
self._assert_stored(ftp_server.path / filename, b"foobar")
@deferred_f_from_coro_f
@coroutine_test
async def test_overwrite(self):
with MockFTPServer() as ftp_server:
filename = "file"
@ -211,7 +211,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar")
self._assert_stored(ftp_server.path / filename, b"bar")
@deferred_f_from_coro_f
@coroutine_test
async def test_append_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -222,7 +222,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", feed_options=feed_options, settings=settings)
self._assert_stored(ftp_server.path / filename, b"foobar")
@deferred_f_from_coro_f
@coroutine_test
async def test_overwrite_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -314,7 +314,7 @@ class TestS3FeedStorage:
assert storage.access_key == "uri_key"
assert storage.secret_key == "uri_secret"
@deferred_f_from_coro_f
@coroutine_test
async def test_store(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
@ -455,7 +455,7 @@ class TestS3FeedStorage:
assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name
@deferred_f_from_coro_f
@coroutine_test
async def test_store_without_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv",
@ -475,7 +475,7 @@ class TestS3FeedStorage:
)
assert acl is None
@deferred_f_from_coro_f
@coroutine_test
async def test_store_with_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
@ -540,7 +540,7 @@ class TestGCSFeedStorage:
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.acl is None
@deferred_f_from_coro_f
@coroutine_test
async def test_store(self):
try:
from google.cloud.storage import Client # noqa: F401,PLC0415
@ -1006,7 +1006,7 @@ class TestFeedExport(TestFeedExportBase):
result = self._load_until_eof(data["marshal"], load_func=marshal.load)
assert result == expected
@inlineCallbacks
@inline_callbacks_test
def test_stats_file_success(self):
settings = {
"FEEDS": {
@ -1020,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@inlineCallbacks
@inline_callbacks_test
def test_stats_file_failed(self):
settings = {
"FEEDS": {
@ -1038,7 +1038,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@inlineCallbacks
@inline_callbacks_test
def test_stats_multiple_file(self):
settings = {
"FEEDS": {
@ -1060,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase):
crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1
)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items(self):
# feed exporters use field names from Item
items = [
@ -1074,7 +1074,7 @@ class TestFeedExport(TestFeedExportBase):
header = self.MyItem.fields.keys()
await self.assertExported(items, header, rows)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):
settings = {
@ -1086,7 +1086,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings)
assert data[fmt] is None
@deferred_f_from_coro_f
@coroutine_test
async def test_start_finish_exporting_items(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1106,7 +1106,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@deferred_f_from_coro_f
@coroutine_test
async def test_start_finish_exporting_no_items(self):
items = []
settings = {
@ -1124,7 +1124,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@deferred_f_from_coro_f
@coroutine_test
async def test_start_finish_exporting_items_exception(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1145,7 +1145,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@deferred_f_from_coro_f
@coroutine_test
async def test_start_finish_exporting_no_items_exception(self):
items = []
settings = {
@ -1164,7 +1164,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@deferred_f_from_coro_f
@coroutine_test
async def test_export_no_items_store_empty(self):
formats = (
("json", b"[]"),
@ -1184,7 +1184,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings)
assert expctd == data[fmt]
@deferred_f_from_coro_f
@coroutine_test
async def test_export_no_items_multiple_feeds(self):
"""Make sure that `storage.store` is called for every feed."""
settings = {
@ -1202,7 +1202,7 @@ class TestFeedExport(TestFeedExportBase):
assert str(log).count("Storage.store is called") == 0
@deferred_f_from_coro_f
@coroutine_test
async def test_export_multiple_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1224,7 +1224,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, header, rows_csv)
await self.assertExportedJsonLines(items, rows_jl)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_empty_field_list(self):
# FEED_EXPORT_FIELDS==[] means the same as default None
items = [{"foo": "bar"}]
@ -1234,7 +1234,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, header, rows)
await self.assertExportedJsonLines(items, rows, settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1242,7 +1242,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_comma_separated_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1250,7 +1250,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": ",".join(header)}
await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_json_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1258,7 +1258,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_field_names(self):
items = [{"foo": "bar"}]
header = {"foo": "Foo"}
@ -1266,7 +1266,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, list(header.values()), rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_dict_field_names(self):
items = [{"foo": "bar"}]
header = {
@ -1277,7 +1277,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items_json_field_names(self):
items = [{"foo": "bar"}]
header = {"foo": "Foo"}
@ -1285,7 +1285,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
await self.assertExported(items, list(header.values()), rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_based_on_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1331,7 +1331,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_export_based_on_custom_filters(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1390,7 +1390,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as
# a header for CSV, and all fields are used for JSON Lines.
@ -1403,7 +1403,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, ["foo", "egg"], rows_csv)
await self.assertExportedJsonLines(items, rows_jl)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_tuple(self):
items = [
{"foo": "bar1", "egg": "spam1"},
@ -1414,7 +1414,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}]
await self.assertExported(items, ["foo", "baz"], rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_feed_export_fields(self):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
@ -1440,7 +1440,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}]
await self.assertExported(items, ["egg", "baz"], rows, settings=settings)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_encoding(self):
items = [{"foo": "Test\xd6"}]
@ -1485,7 +1485,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings)
assert data[fmt] == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_export_multiple_configs(self):
items = [{"foo": "FOO", "bar": "BAR"}]
@ -1525,7 +1525,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_export_indentation(self):
items = [
{"foo": ["bar"]},
@ -1681,7 +1681,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings)
assert data[row["format"]] == row["expected"]
@deferred_f_from_coro_f
@coroutine_test
async def test_init_exporters_storages_with_crawler(self):
settings = {
"FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter},
@ -1694,7 +1694,7 @@ class TestFeedExport(TestFeedExportBase):
assert FromCrawlerCsvItemExporter.init_with_crawler
assert FromCrawlerFileFeedStorage.init_with_crawler
@deferred_f_from_coro_f
@coroutine_test
async def test_str_uri(self):
settings = {
"FEED_STORE_EMPTY": True,
@ -1703,7 +1703,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings)
assert data["csv"] == b""
@deferred_f_from_coro_f
@coroutine_test
async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
settings = {
"FEEDS": {
@ -1723,7 +1723,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]:
assert f"Stored {fmt} feed (2 items)" in str(log)
@deferred_f_from_coro_f
@coroutine_test
async def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
settings = {
"FEEDS": {
@ -1743,7 +1743,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]:
assert f"Error storing {fmt} feed (2 items)" in str(log)
@deferred_f_from_coro_f
@coroutine_test
async def test_extend_kwargs(self):
items = [{"foo": "FOO", "bar": "BAR"}]
@ -1780,7 +1780,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings)
assert data[feed_options["format"]] == row["expected"]
@deferred_f_from_coro_f
@coroutine_test
async def test_storage_file_no_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
@ -1802,7 +1802,7 @@ class TestFeedExport(TestFeedExportBase):
await self.exported_no_data(settings)
assert Storage.open_file is Storage.store_file
@deferred_f_from_coro_f
@coroutine_test
async def test_storage_file_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
@ -1901,7 +1901,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data_stream.seek(0)
return data_stream.read()
@deferred_f_from_coro_f
@coroutine_test
async def test_gzip_plugin(self):
filename = self._named_tempfile("gzip_file")
@ -1920,7 +1920,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError:
pytest.fail("Received invalid gzip data.")
@deferred_f_from_coro_f
@coroutine_test
async def test_gzip_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
@ -1957,7 +1957,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_gzip_plugin_mtime(self):
filename_to_compressed = {
self._named_tempfile("mtime_123"): self.get_gzip_compressed(
@ -1992,7 +1992,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_gzip_plugin_filename(self):
filename_to_compressed = {
self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
@ -2027,7 +2027,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_lzma_plugin(self):
filename = self._named_tempfile("lzma_file")
@ -2046,7 +2046,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except lzma.LZMAError:
pytest.fail("Received invalid lzma data.")
@deferred_f_from_coro_f
@coroutine_test
async def test_lzma_plugin_format(self):
filename_to_compressed = {
self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
@ -2079,7 +2079,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_lzma_plugin_check(self):
filename_to_compressed = {
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
@ -2112,7 +2112,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_lzma_plugin_preset(self):
filename_to_compressed = {
self._named_tempfile("preset_PRESET_0"): lzma.compress(
@ -2145,7 +2145,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_lzma_plugin_filters(self):
if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
@ -2170,7 +2170,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = lzma.decompress(data[filename])
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_bz2_plugin(self):
filename = self._named_tempfile("bz2_file")
@ -2189,7 +2189,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError:
pytest.fail("Received invalid bz2 data.")
@deferred_f_from_coro_f
@coroutine_test
async def test_bz2_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_1"): bz2.compress(
@ -2222,7 +2222,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_custom_plugin(self):
filename = self._named_tempfile("csv_file")
@ -2238,7 +2238,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = await self.exported_data(self.items, settings)
assert data[filename] == self.expected
@deferred_f_from_coro_f
@coroutine_test
async def test_custom_plugin_with_parameter(self):
expected = b"foo\r\n\nbar\r\n\n"
filename = self._named_tempfile("newline")
@ -2256,7 +2256,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = await self.exported_data(self.items, settings)
assert data[filename] == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_custom_plugin_with_compression(self):
expected = b"foo\r\n\nbar\r\n\n"
@ -2301,7 +2301,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = decompressor(data[filename])
assert result == expected
@deferred_f_from_coro_f
@coroutine_test
async def test_exports_compatibility_with_postproc(self):
filename_to_expected = {
self._named_tempfile("csv"): b"foo\r\nbar\r\n",
@ -2511,7 +2511,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@deferred_f_from_coro_f
@coroutine_test
async def test_export_items(self):
"""Test partial deliveries in all supported formats"""
items = [
@ -2540,7 +2540,7 @@ class TestBatchDeliveries(TestFeedExportBase):
with pytest.raises(NotConfigured):
FeedExporter(crawler)
@deferred_f_from_coro_f
@coroutine_test
async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):
settings = {
@ -2556,7 +2556,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data)
assert len(data[fmt]) == 0
@deferred_f_from_coro_f
@coroutine_test
async def test_export_no_items_store_empty(self):
formats = (
("json", b"[]"),
@ -2580,7 +2580,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data)
assert data[fmt][0] == expctd
@deferred_f_from_coro_f
@coroutine_test
async def test_export_multiple_configs(self):
items = [
{"foo": "FOO", "bar": "BAR"},
@ -2636,7 +2636,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@deferred_f_from_coro_f
@coroutine_test
async def test_batch_item_count_feeds_setting(self):
items = [{"foo": "FOO"}, {"foo": "FOO1"}]
formats = {
@ -2660,7 +2660,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@deferred_f_from_coro_f
@coroutine_test
async def test_batch_path_differ(self):
"""
Test that the name of all batch files differ from each other.
@ -2682,7 +2682,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = await self.exported_data(items, settings)
assert len(items) == len(data["json"])
@inlineCallbacks
@inline_callbacks_test
def test_stats_batch_file_success(self):
settings = {
"FEEDS": {
@ -2700,7 +2700,7 @@ class TestBatchDeliveries(TestFeedExportBase):
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@pytest.mark.requires_boto3
@inlineCallbacks
@inline_callbacks_test
def test_s3_export(self):
bucket = "mybucket"
items = [
@ -2823,7 +2823,7 @@ class TestFeedExporterSignals:
feed_exporter.item_scraped(item, spider)
await feed_exporter.close_spider(spider)
@deferred_f_from_coro_f
@coroutine_test
async def test_feed_exporter_signals_sent(self) -> None:
self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False
@ -2835,7 +2835,7 @@ class TestFeedExporterSignals:
assert self.feed_slot_closed_received
assert self.feed_exporter_closed_received
@deferred_f_from_coro_f
@coroutine_test
async def test_feed_exporter_signals_sent_async(self) -> None:
self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False

6
tests/test_logformatter.py
View File

@ -12,7 +12,7 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class CustomItem(Item):
@ -272,7 +272,7 @@ class TestShowOrSkipMessages:
},
}
@inlineCallbacks
@inline_callbacks_test
def test_show_messages(self):
crawler = get_crawler(ItemSpider, self.base_settings)
with LogCapture() as lc:
@ -281,7 +281,7 @@ class TestShowOrSkipMessages:
assert "Crawled (200) <GET http://127.0.0.1:" in str(lc)
assert "Dropped: Ignoring item" in str(lc)
@inlineCallbacks
@inline_callbacks_test
def test_skip_messages(self):
settings = self.base_settings.copy()
settings["LOG_FORMATTER"] = SkipMessagesLogFormatter

12
tests/test_pipeline_crawl.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@ -145,7 +145,7 @@ class TestFileDownloadCrawl:
# check that no files were written to the media store
assert not list(self.tmpmediastore.iterdir())
@inlineCallbacks
@inline_callbacks_test
def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log:
@ -156,7 +156,7 @@ class TestFileDownloadCrawl:
)
self._assert_files_downloaded(self.items, str(log))
@inlineCallbacks
@inline_callbacks_test
def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log:
@ -167,7 +167,7 @@ class TestFileDownloadCrawl:
)
self._assert_files_download_failure(crawler, self.items, 404, str(log))
@inlineCallbacks
@inline_callbacks_test
def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
@ -179,7 +179,7 @@ class TestFileDownloadCrawl:
)
self._assert_files_download_failure(crawler, self.items, 302, str(log))
@inlineCallbacks
@inline_callbacks_test
def test_download_media_redirected_allowed(self):
settings = {
**self.settings,
@ -196,7 +196,7 @@ class TestFileDownloadCrawl:
self._assert_files_downloaded(self.items, str(log))
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@inlineCallbacks
@inline_callbacks_test
def test_download_media_file_path_error(self):
cls = load_object(self.pipeline_class)

18
tests/test_pipeline_files.py
View File

@ -34,7 +34,7 @@ from scrapy.settings import Settings
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from .test_pipeline_media import _mocked_download_func
@ -161,7 +161,7 @@ class TestFilesPipeline:
fullpath = Path(self.tempdir, "some", "image", "key.jpg")
assert self.pipeline.store._get_filesystem_path(path) == fullpath
@deferred_f_from_coro_f
@coroutine_test
async def test_file_not_expired(self):
item_url = "http://example.com/file.pdf"
item = _create_item_with_files(item_url)
@ -188,7 +188,7 @@ class TestFilesPipeline:
for p in patchers:
p.stop()
@deferred_f_from_coro_f
@coroutine_test
async def test_file_expired(self):
item_url = "http://example.com/file2.pdf"
item = _create_item_with_files(item_url)
@ -219,7 +219,7 @@ class TestFilesPipeline:
for p in patchers:
p.stop()
@deferred_f_from_coro_f
@coroutine_test
async def test_file_cached(self):
item_url = "http://example.com/file3.pdf"
item = _create_item_with_files(item_url)
@ -563,7 +563,7 @@ class TestFilesPipelineCustomSettings:
@pytest.mark.requires_botocore
class TestS3FilesStore:
@inlineCallbacks
@inline_callbacks_test
def test_persist(self):
bucket = "mybucket"
key = "export.csv"
@ -603,7 +603,7 @@ class TestS3FilesStore:
# The call to read does not happen with Stubber
assert buffer.method_calls == [mock.call.seek(0)]
@inlineCallbacks
@inline_callbacks_test
def test_stat(self):
bucket = "mybucket"
key = "export.csv"
@ -640,7 +640,7 @@ class TestS3FilesStore:
"GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
)
class TestGCSFilesStore:
@inlineCallbacks
@inline_callbacks_test
def test_persist(self):
uri = os.environ.get("GCS_TEST_FILE_URI")
if not uri:
@ -665,7 +665,7 @@ class TestGCSFilesStore:
assert blob.content_type == "application/octet-stream"
assert expected_policy in acl
@inlineCallbacks
@inline_callbacks_test
def test_blob_path_consistency(self):
"""Test to make sure that paths used to store files is the same as the one used to get
already uploaded files.
@ -693,7 +693,7 @@ class TestGCSFilesStore:
@pytest.mark.requires_reactor # needs a reactor for FTPFilesStore
class TestFTPFileStore:
@inlineCallbacks
@inline_callbacks_test
def test_persist(self):
data = b"TestFTPFilesStore: \xe2\x98\x83"
buf = BytesIO(data)

22
tests/test_pipeline_media.py
View File

@ -18,7 +18,7 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
async def _mocked_download_func(request):
@ -163,7 +163,7 @@ class TestBaseMediaPipeline:
assert new_item is item
assert len(log.records) == 0
@deferred_f_from_coro_f
@coroutine_test
async def test_default_process_item(self):
item = {"name": "name"}
new_item = await self.pipe.process_item(item)
@ -211,7 +211,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
self.pipe._mockcalled.append("request_errback")
return result
@deferred_f_from_coro_f
@coroutine_test
async def test_result_succeed(self):
rsp = Response("http://url1")
req = Request(
@ -229,7 +229,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
"item_completed",
]
@deferred_f_from_coro_f
@coroutine_test
async def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo")
@ -252,7 +252,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
"item_completed",
]
@deferred_f_from_coro_f
@coroutine_test
async def test_mix_of_success_and_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
rsp1 = Response("http://url1")
@ -278,7 +278,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert m.count("media_downloaded") == 1
assert m.count("media_failed") == 1
@deferred_f_from_coro_f
@coroutine_test
async def test_get_media_requests(self):
# returns single Request (without callback)
req = Request("http://url")
@ -296,7 +296,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) in self.info.downloaded
assert self.fingerprint(req2) in self.info.downloaded
@deferred_f_from_coro_f
@coroutine_test
async def test_results_are_cached_across_multiple_items(self):
rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1})
@ -315,7 +315,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})]
@deferred_f_from_coro_f
@coroutine_test
async def test_results_are_cached_for_requests_of_single_item(self):
rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1})
@ -327,7 +327,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert new_item is item
assert new_item["results"] == [(True, {}), (True, {})]
@deferred_f_from_coro_f
@coroutine_test
async def test_wait_if_request_is_downloading(self):
def _check_downloading(response):
fp = self.fingerprint(req1)
@ -352,7 +352,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
new_item = await self.pipe.process_item(item)
assert new_item["results"] == [(True, {}), (True, {})]
@deferred_f_from_coro_f
@coroutine_test
async def test_use_media_to_download_result(self):
req = Request("http://url", meta={"result": "ITSME"})
item = {"requests": req}
@ -480,7 +480,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):
self.pipe._mockcalled.append("request_errback")
return result
@deferred_f_from_coro_f
@coroutine_test
async def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo")

26
tests/test_pipelines.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class SimplePipeline:
@ -154,13 +154,13 @@ class TestPipeline:
),
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None:
crawler = self._create_crawler(pipeline_class)
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@deferred_f_from_coro_f
@coroutine_test
async def test_pipeline_deferred(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeferredPipeline)
with (
@ -180,7 +180,7 @@ class TestPipeline:
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeprecatedSpiderArgPipeline)
with (
@ -214,7 +214,7 @@ class TestPipeline:
ProcessItemExceptionAsyncPipeline,
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_process_item_exception(
self,
caplog: pytest.LogCaptureFixture,
@ -239,7 +239,7 @@ class TestPipeline:
OpenSpiderExceptionAsyncPipeline,
],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_open_spider_exception(
self, mockserver: MockServer, pipeline_class: type
) -> None:
@ -267,7 +267,7 @@ class TestCustomPipelineManager:
itemproc.process_item({}, crawler.spider)
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_integration_recommended(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
async def process_item_async(self, item):
@ -294,7 +294,7 @@ class TestCustomPipelineManager:
assert len(items) == 1
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
def open_spider(self, spider):
@ -353,7 +353,7 @@ class TestCustomPipelineManager:
assert len(items) == 1
@pytest.mark.requires_http_handler
@deferred_f_from_coro_f
@coroutine_test
async def test_integration_no_async_not_subclass(
self, mockserver: MockServer
) -> None:
@ -425,7 +425,7 @@ class TestMiddlewareManagerSpider:
def crawler(self) -> Crawler:
return get_crawler(Spider)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg_no_crawler_spider(
self, crawler: Crawler
) -> None:
@ -480,7 +480,7 @@ class TestMiddlewareManagerSpider:
):
await mwman.close_spider_async()
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to a deprecated method
is ignored, even if mismatched."""
@ -497,7 +497,7 @@ class TestMiddlewareManagerSpider:
):
await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg_without_crawler(self) -> None:
"""The first instance passed to a deprecated method is used. Mismatched ones raise an error."""
with pytest.warns(
@ -527,7 +527,7 @@ class TestMiddlewareManagerSpider:
):
await maybe_deferred_to_future(mwman.close_spider(spider))
@deferred_f_from_coro_f
@coroutine_test
async def test_no_spider_arg_without_crawler(self) -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(

8
tests/test_proxy_connect.py
View File

@ -13,7 +13,7 @@ from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider, SingleRequestSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class MitmProxy:
@ -84,14 +84,14 @@ class TestProxyConnect:
self._proxy.stop()
os.environ = self._oldenv
@inlineCallbacks
@inline_callbacks_test
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, log)
@inlineCallbacks
@inline_callbacks_test
def test_https_tunnel_auth_error(self):
os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
crawler = get_crawler(SimpleSpider)
@ -101,7 +101,7 @@ class TestProxyConnect:
# he just sees a TunnelError.
self._assert_got_tunnel_error(log)
@inlineCallbacks
@inline_callbacks_test
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)

16
tests/test_request_attribute_binding.py
View File

@ -6,7 +6,7 @@ from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SingleRequestSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
OVERRIDDEN_URL = "https://example.org"
@ -74,7 +74,7 @@ class TestCrawl:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_response_200(self):
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(SingleRequestSpider)
@ -82,7 +82,7 @@ class TestCrawl:
response = crawler.spider.meta["responses"][0]
assert response.request.url == url
@inlineCallbacks
@inline_callbacks_test
def test_response_error(self):
for status in ("404", "500"):
url = self.mockserver.url(f"/status?n={status}")
@ -93,7 +93,7 @@ class TestCrawl:
assert failure.request.url == url
assert response.request.url == url
@inlineCallbacks
@inline_callbacks_test
def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(
@ -109,7 +109,7 @@ class TestCrawl:
assert failure.request.url == url
assert isinstance(failure.value, ZeroDivisionError)
@inlineCallbacks
@inline_callbacks_test
def test_downloader_middleware_override_request_in_process_response(self):
"""
Downloader middleware which returns a response with an specific 'request' attribute.
@ -152,7 +152,7 @@ class TestCrawl:
),
)
@inlineCallbacks
@inline_callbacks_test
def test_downloader_middleware_override_in_process_exception(self):
"""
An exception is raised but caught by the next middleware, which
@ -175,7 +175,7 @@ class TestCrawl:
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == OVERRIDDEN_URL
@inlineCallbacks
@inline_callbacks_test
def test_downloader_middleware_do_not_override_in_process_exception(self):
"""
An exception is raised but caught by the next middleware, which
@ -198,7 +198,7 @@ class TestCrawl:
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == url
@inlineCallbacks
@inline_callbacks_test
def test_downloader_middleware_alternative_callback(self):
"""
Downloader middleware which returns a response with a

4
tests/test_request_cb_kwargs.py
View File

@ -5,7 +5,7 @@ from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class InjectArgumentsDownloaderMiddleware:
@ -160,7 +160,7 @@ class TestCallbackKeywordArguments:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_callback_kwargs(self):
crawler = get_crawler(KeywordArgumentsSpider)
with LogCapture() as log:

10
tests/test_request_left.py
View File

@ -2,7 +2,7 @@ from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class SignalCatcherSpider(Spider):
@ -32,25 +32,25 @@ class TestCatching:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_success(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/status?n=200"))
assert crawler.spider.caught_times == 1
@inlineCallbacks
@inline_callbacks_test
def test_timeout(self):
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
assert crawler.spider.caught_times == 1
@inlineCallbacks
@inline_callbacks_test
def test_disconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/drop"))
assert crawler.spider.caught_times == 1
@inlineCallbacks
@inline_callbacks_test
def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")

4
tests/test_scheduler.py
View File

@ -19,7 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class MemoryScheduler(BaseScheduler):
@ -370,7 +370,7 @@ class TestIntegrationWithDownloaderAwareInMemory:
)
@pytest.mark.requires_http_handler
@inlineCallbacks
@inline_callbacks_test
def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver:
url = mockserver.url("/status?n=200", is_secure=False)

6
tests/test_scheduler_base.py
View File

@ -13,7 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.request import fingerprint
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
PATHS = ["/a", "/b", "/c"]
URLS = [urljoin("https://example.org", p) for p in PATHS]
@ -118,7 +118,7 @@ class TestSimpleScheduler(InterfaceCheckMixin):
def setup_method(self):
self.scheduler = SimpleScheduler()
@inlineCallbacks
@inline_callbacks_test
def test_enqueue_dequeue(self):
open_result = yield self.scheduler.open(Spider("foo"))
assert open_result == "open"
@ -148,7 +148,7 @@ class TestSimpleScheduler(InterfaceCheckMixin):
class TestMinimalSchedulerCrawl:
scheduler_cls = MinimalScheduler
@inlineCallbacks
@inline_callbacks_test
def test_crawl(self):
with MockServer() as mockserver:
settings = {

6
tests/test_signals.py
View File

@ -3,7 +3,7 @@ import pytest
from scrapy import Request, Spider, signals
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
class ItemSpider(Spider):
@ -20,7 +20,7 @@ class ItemSpider(Spider):
class TestMain:
@deferred_f_from_coro_f
@coroutine_test
async def test_scheduler_empty(self):
crawler = get_crawler()
calls = []
@ -52,7 +52,7 @@ class TestMockServer:
@pytest.mark.requires_http_handler
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_simple_pipeline(self):
crawler = get_crawler(ItemSpider)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)

10
tests/test_spider.py
View File

@ -30,7 +30,7 @@ from scrapy.spiders import (
from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from tests.utils.decorators import coroutine_test, inline_callbacks_test
class TestSpider:
@ -95,7 +95,7 @@ class TestSpider:
assert settings.get("TEST2") == "spider"
assert settings.get("TEST3") == "project"
@inlineCallbacks
@inline_callbacks_test
def test_settings_in_from_crawler(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {
@ -142,7 +142,7 @@ class TestSpider:
class TestInitSpider(TestSpider):
spider_class = InitSpider
@deferred_f_from_coro_f
@coroutine_test
async def test_start_urls(self):
responses = []
@ -458,7 +458,7 @@ class TestCrawlSpider(TestSpider):
assert hasattr(spider, "_follow_links")
assert not spider._follow_links
@inlineCallbacks
@inline_callbacks_test
def test_start_url(self):
class TestSpider(self.spider_class):
name = "test"
@ -827,7 +827,7 @@ Sitemap: /sitemap-relative-url.xml
),
)
@deferred_f_from_coro_f
@coroutine_test
async def test_sitemap_urls(self):
class TestSpider(self.spider_class):
name = "test"

24
tests/test_spider_start.py
View File

@ -13,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
from .utils.decorators import deferred_f_from_coro_f
from .utils.decorators import coroutine_test
SLEEP_SECONDS = 0.1
@ -38,7 +38,7 @@ class TestMain:
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items
@deferred_f_from_coro_f
@coroutine_test
async def test_start_urls(self):
class TestSpider(Spider):
name = "test"
@ -51,7 +51,7 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_start(self):
class TestSpider(Spider):
name = "test"
@ -63,7 +63,7 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_start_subclass(self):
class BaseSpider(Spider):
async def start(self):
@ -76,7 +76,7 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated(self):
class TestSpider(Spider):
name = "test"
@ -87,7 +87,7 @@ class TestMain:
with pytest.warns(ScrapyDeprecationWarning):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_subclass(self):
class BaseSpider(Spider):
def start_requests(self):
@ -100,7 +100,7 @@ class TestMain:
with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_universal(self):
class TestSpider(Spider):
name = "test"
@ -115,7 +115,7 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_subclass(self):
class BaseSpider(Spider):
async def start(self):
@ -131,7 +131,7 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
@coroutine_test
async def test_start_deprecated_super(self):
class TestSpider(Spider):
name = "test"
@ -154,7 +154,7 @@ class TestMain:
await self._test_spider(TestSpider, expected_items)
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncio_delayed(self):
async def start(spider):
await sleep(SLEEP_SECONDS)
@ -163,7 +163,7 @@ class TestMain:
await self._test_start(start, [ITEM_A])
@pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
@deferred_f_from_coro_f
@coroutine_test
async def test_twisted_delayed(self):
async def start(spider):
await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
@ -173,7 +173,7 @@ class TestMain:
# Exceptions
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_non_generator_exception(self):
class TestSpider(Spider):
name = "test"

90
tests/test_spidermiddleware.py
View File

@ -18,7 +18,7 @@ from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.python.failure import Failure
@ -53,7 +53,7 @@ class TestSpiderMiddleware:
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_input method"""
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response):
@ -67,7 +67,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_output method"""
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result):
@ -81,7 +81,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_exception method"""
@deferred_f_from_coro_f
@coroutine_test
async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception):
@ -100,7 +100,7 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
"""Re raise the exception by returning None"""
@deferred_f_from_coro_f
@coroutine_test
async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception):
@ -237,47 +237,47 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
@deferred_f_from_coro_f
@coroutine_test
async def test_simple(self):
"""Simple mw"""
await self._test_simple_base(self.MW_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncgen(self):
"""Asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_asyncgen(self):
"""Simple mw -> asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncgen_simple(self):
"""Asyncgen mw -> simple mw; upgrade then downgrade"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal(self):
"""Universal mw"""
await self._test_simple_base(self.MW_UNIVERSAL)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_simple(self):
"""Universal mw -> simple mw"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_universal(self):
"""Simple mw -> universal mw"""
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_asyncgen(self):
"""Universal mw -> asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncgen_universal(self):
"""Asyncgen mw -> universal mw; upgrade"""
await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
@ -290,27 +290,27 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
for item in super()._callback():
yield item
@deferred_f_from_coro_f
@coroutine_test
async def test_simple(self):
"""Simple mw; downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_asyncgen(self):
"""Simple mw -> asyncgen mw; downgrade then upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal(self):
"""Universal mw"""
await self._test_asyncgen_base(self.MW_UNIVERSAL)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_simple(self):
"""Universal mw -> simple mw; downgrade"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_universal(self):
"""Simple mw -> universal mw; downgrade"""
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
@ -327,7 +327,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
@deferred_f_from_coro_f
@coroutine_test
async def test_non_iterable(self):
with pytest.raises(
_InvalidOutput,
@ -335,7 +335,7 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
):
await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware)
@deferred_f_from_coro_f
@coroutine_test
async def test_coroutine(self):
with pytest.raises(
_InvalidOutput,
@ -375,7 +375,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
return await self.mwman.process_start()
@deferred_f_from_coro_f
@coroutine_test
async def test_simple(self):
"""Simple mw"""
start = await self._get_processed_start(self.MW_SIMPLE)
@ -489,33 +489,33 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
self._scrape_func, self.response, self.request
)
@deferred_f_from_coro_f
@coroutine_test
async def test_just_builtin(self):
await self._test_simple_base()
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_simple(self):
await self._test_simple_base(self.MW_SIMPLE, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_async(self):
"""Upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_universal(self):
await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_builtin(self):
await self._test_simple_base(self.MW_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_async_builtin(self):
"""Upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_builtin(self):
await self._test_simple_base(self.MW_UNIVERSAL)
@ -525,33 +525,33 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
for item in super()._callback():
yield item
@deferred_f_from_coro_f
@coroutine_test
async def test_just_builtin(self):
await self._test_asyncgen_base()
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_simple(self):
"""Downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_async(self):
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_builtin_universal(self):
await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
@deferred_f_from_coro_f
@coroutine_test
async def test_simple_builtin(self):
"""Downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f
@coroutine_test
async def test_async_builtin(self):
await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_builtin(self):
await self._test_asyncgen_base(self.MW_UNIVERSAL)
@ -574,39 +574,39 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
):
await self._get_middleware_result(*mw_classes)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_simple(self):
"""Simple exc mw"""
await self._test_simple_base(self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_async(self):
"""Async exc mw"""
await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_simple_simple(self):
"""Simple exc mw -> simple output mw"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_async_async(self):
"""Async exc mw -> async output mw"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_simple_async(self):
"""Simple exc mw -> async output mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f
@coroutine_test
async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
class TestDeprecatedSpiderArg(TestSpiderMiddleware):
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mw_spider_arg(self):
class DeprecatedSpiderArgMiddleware:
def process_spider_input(self, response, spider):
@ -635,7 +635,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware):
self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
await self._scrape_response()
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mwman_spider_arg(self):
with pytest.warns(
ScrapyDeprecationWarning,
@ -644,7 +644,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware):
):
await self.mwman.process_start(DefaultSpider())
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mwman_spider_arg_no_crawler(self):
with pytest.warns(
ScrapyDeprecationWarning,

8
tests/test_spidermiddleware_httperror.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class _HttpErrorSpider(MockServerSpider):
@ -202,7 +202,7 @@ class TestHttpErrorMiddlewareIntegrational:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
@inline_callbacks_test
def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -216,7 +216,7 @@ class TestHttpErrorMiddlewareIntegrational:
assert get_value("httperror/response_ignored_status_count/402") == 1
assert get_value("httperror/response_ignored_status_count/500") == 1
@inlineCallbacks
@inline_callbacks_test
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture() as log:
@ -230,7 +230,7 @@ class TestHttpErrorMiddlewareIntegrational:
assert "Ignoring response <200" not in str(log)
assert "Ignoring response <402" not in str(log)
@inlineCallbacks
@inline_callbacks_test
def test_logging_level(self):
# HttpError logs ignored responses with level INFO
crawler = get_crawler(_HttpErrorSpider)

24
tests/test_spidermiddleware_output_chain.py
View File

@ -4,7 +4,7 @@ from testfixtures import LogCapture
from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class _BaseSpiderMiddleware:
@ -338,7 +338,7 @@ class TestSpiderMiddleware:
await crawler.crawl_async(mockserver=self.mockserver)
return log
@deferred_f_from_coro_f
@coroutine_test
async def test_recovery(self):
"""
(0) Recover from an exception in a spider callback. The final item count should be 3
@ -351,7 +351,7 @@ class TestSpiderMiddleware:
assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log)
@deferred_f_from_coro_f
@coroutine_test
async def test_recovery_asyncgen(self):
"""
Same as test_recovery but with an async callback.
@ -361,7 +361,7 @@ class TestSpiderMiddleware:
assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log)
@deferred_f_from_coro_f
@coroutine_test
async def test_process_spider_input_without_errback(self):
"""
(1.1) An exception from the process_spider_input chain should be caught by the
@ -371,7 +371,7 @@ class TestSpiderMiddleware:
assert "Middleware: will raise IndexError" in str(log1)
assert "Middleware: IndexError exception caught" in str(log1)
@deferred_f_from_coro_f
@coroutine_test
async def test_process_spider_input_with_errback(self):
"""
(1.2) An exception from the process_spider_input chain should not be caught by the
@ -385,7 +385,7 @@ class TestSpiderMiddleware:
assert "{'from': 'callback'}" not in str(log1)
assert "'item_scraped_count': 1" in str(log1)
@deferred_f_from_coro_f
@coroutine_test
async def test_generator_callback(self):
"""
(2) An exception from a spider callback (returning a generator) should
@ -396,7 +396,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2)
@deferred_f_from_coro_f
@coroutine_test
async def test_async_generator_callback(self):
"""
Same as test_generator_callback but with an async callback.
@ -405,7 +405,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2)
@deferred_f_from_coro_f
@coroutine_test
async def test_generator_callback_right_after_callback(self):
"""
(2.1) Special case of (2): Exceptions should be caught
@ -415,7 +415,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log21)
assert "'item_scraped_count': 2" in str(log21)
@deferred_f_from_coro_f
@coroutine_test
async def test_not_a_generator_callback(self):
"""
(3) An exception from a spider callback (returning a list) should
@ -425,7 +425,7 @@ class TestSpiderMiddleware:
assert "Middleware: ZeroDivisionError exception caught" in str(log3)
assert "item_scraped_count" not in str(log3)
@deferred_f_from_coro_f
@coroutine_test
async def test_not_a_generator_callback_right_after_callback(self):
"""
(3.1) Special case of (3): Exceptions should be caught
@ -437,7 +437,7 @@ class TestSpiderMiddleware:
assert "Middleware: ZeroDivisionError exception caught" in str(log31)
assert "item_scraped_count" not in str(log31)
@deferred_f_from_coro_f
@coroutine_test
async def test_generator_output_chain(self):
"""
(4) An exception from a middleware's process_spider_output method should be sent
@ -484,7 +484,7 @@ class TestSpiderMiddleware:
assert str(item_recovered) in str(log4)
assert "parse-second-item" not in str(log4)
@deferred_f_from_coro_f
@coroutine_test
async def test_not_a_generator_output_chain(self):
"""
(5) An exception from a middleware's process_spider_output method should be sent

48
tests/test_spidermiddleware_process_start.py
View File

@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler
from tests.test_spider_start import SLEEP_SECONDS
from .utils import twisted_sleep
from .utils.decorators import deferred_f_from_coro_f
from .utils.decorators import coroutine_test
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
@ -130,45 +130,45 @@ class TestMain:
expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C]
await self._test([smw1, smw2], spider_cls, expected_items)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_modern_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_universal_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_deprecated_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
):
await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_modern_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_universal_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_deprecated_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
):
await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mw_modern_spider(self):
with (
pytest.warns(
@ -180,7 +180,7 @@ class TestMain:
):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mw_modern_spider_subclass(self):
with (
pytest.warns(
@ -195,7 +195,7 @@ class TestMain:
DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass
)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mw_universal_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
@ -206,7 +206,7 @@ class TestMain:
[ITEM_A, ITEM_D, ITEM_C],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_mw_deprecated_spider(self):
with (
pytest.warns(
@ -218,7 +218,7 @@ class TestMain:
):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_universal_mw_modern_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
@ -228,7 +228,7 @@ class TestMain:
ModernWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_deprecated_mw_modern_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap(
@ -237,7 +237,7 @@ class TestMain:
ModernWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_deprecated_mw_modern_spider(self):
with (
pytest.warns(
@ -253,7 +253,7 @@ class TestMain:
ModernWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_universal_mw_universal_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
@ -263,7 +263,7 @@ class TestMain:
UniversalWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_deprecated_mw_universal_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap(
@ -272,7 +272,7 @@ class TestMain:
UniversalWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_deprecated_mw_universal_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
@ -284,7 +284,7 @@ class TestMain:
[ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],
)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_universal_mw_deprecated_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
@ -295,7 +295,7 @@ class TestMain:
DeprecatedWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_modern_mw_deprecated_mw_deprecated_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap(
@ -304,7 +304,7 @@ class TestMain:
DeprecatedWrapSpider,
)
@deferred_f_from_coro_f
@coroutine_test
async def test_universal_mw_deprecated_mw_deprecated_spider(self):
with (
pytest.warns(
@ -330,24 +330,24 @@ class TestMain:
await self._test(spider_middlewares, TestSpider, [ITEM_A])
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncio_sleep_single(self):
await self._test_sleep([AsyncioSleepSpiderMiddleware])
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_asyncio_sleep_multiple(self):
await self._test_sleep(
[NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
)
@pytest.mark.requires_reactor
@deferred_f_from_coro_f
@coroutine_test
async def test_twisted_sleep_single(self):
await self._test_sleep([TwistedSleepSpiderMiddleware])
@pytest.mark.requires_reactor
@deferred_f_from_coro_f
@coroutine_test
async def test_twisted_sleep_multiple(self):
await self._test_sleep(
[NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]

6
tests/test_spidermiddleware_start.py
View File

@ -3,11 +3,11 @@ from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class TestMiddleware:
@deferred_f_from_coro_f
@coroutine_test
async def test_async(self):
crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler)
@ -24,7 +24,7 @@ class TestMiddleware:
]
assert result == [True, True, False, "foo"]
@deferred_f_from_coro_f
@coroutine_test
async def test_sync(self):
crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler)

6
tests/test_stats.py
View File

@ -12,7 +12,7 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@ -121,7 +121,7 @@ class TestStatsCollector:
):
assert stats.get_stats(spider) == {"test": "value"}
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg_custom_collector(self) -> None:
class CustomStatsCollector:
def __init__(self, crawler):
@ -153,7 +153,7 @@ class TestStatsCollector:
):
await crawler.crawl_async(url="data:,")
@deferred_f_from_coro_f
@coroutine_test
async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:
class CustomStatsCollector(StatsCollector):
def open_spider(self, spider): # pylint: disable=signature-differs

6
tests/test_utils_asyncgen.py
View File

@ -1,15 +1,15 @@
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class TestAsyncgenUtils:
@deferred_f_from_coro_f
@coroutine_test
async def test_as_async_generator(self):
ag = as_async_generator(range(42))
results = [i async for i in ag]
assert results == list(range(42))
@deferred_f_from_coro_f
@coroutine_test
async def test_collect_asyncgen(self):
ag = as_async_generator(range(42))
results = await collect_asyncgen(ag)

6
tests/test_utils_asyncio.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.asyncio import (
_parallel_asyncio,
is_asyncio_available,
)
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@ -67,7 +67,7 @@ class TestParallelAsyncio:
await asyncio.sleep(random.random() / 20)
yield i
@deferred_f_from_coro_f
@coroutine_test
async def test_simple(self):
for length in [20, 50, 100]:
parallel_count = [0]
@ -85,7 +85,7 @@ class TestParallelAsyncio:
assert list(range(length)) == sorted(results)
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
@deferred_f_from_coro_f
@coroutine_test
async def test_delays(self):
for length in [20, 50, 100]:
parallel_count = [0]

27
tests/test_utils_defer.py
View File

@ -6,8 +6,7 @@ from asyncio import Future
from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred, succeed
from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
@ -20,7 +19,7 @@ from scrapy.utils.defer import (
mustbe_deferred,
parallel_async,
)
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
@ -29,7 +28,7 @@ if TYPE_CHECKING:
@pytest.mark.requires_reactor
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestMustbeDeferred:
@inlineCallbacks
@inline_callbacks_test
def test_success_function(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = []
@ -45,7 +44,7 @@ class TestMustbeDeferred:
steps.append(2) # add another value, that should be caught by assertEqual
yield dfd
@inlineCallbacks
@inline_callbacks_test
def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = []
@ -224,7 +223,7 @@ class TestParallelAsync:
await maybe_deferred_to_future(dfd)
yield i
@inlineCallbacks
@inline_callbacks_test
def test_simple(self):
for length in [20, 50, 100]:
parallel_count = [0]
@ -244,7 +243,7 @@ class TestParallelAsync:
assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
@inlineCallbacks
@inline_callbacks_test
def test_delays(self):
for length in [20, 50, 100]:
parallel_count = [0]
@ -276,7 +275,7 @@ class TestDeferredFromCoro:
result = deferred_from_coro(42)
assert result == 42
@inlineCallbacks
@inline_callbacks_test
def test_coroutine(self):
async def coroutine() -> int:
return 42
@ -287,7 +286,7 @@ class TestDeferredFromCoro:
assert coro_result == 42
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_coroutine_asyncio(self):
async def coroutine() -> int:
await asyncio.sleep(0.01)
@ -299,7 +298,7 @@ class TestDeferredFromCoro:
assert coro_result == 42
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_future(self):
future = Future()
result = deferred_from_coro(future)
@ -310,7 +309,7 @@ class TestDeferredFromCoro:
class TestDeferredFFromCoroF:
@inlineCallbacks_orig
@inlineCallbacks
def _assert_result(
self, c_f: Callable[[], Awaitable[int]]
) -> Generator[Deferred[Any], Any, None]:
@ -320,7 +319,7 @@ class TestDeferredFFromCoroF:
result = yield d
assert result == 42
@inlineCallbacks
@inline_callbacks_test
def test_coroutine(self):
async def c_f() -> int:
return 42
@ -328,7 +327,7 @@ class TestDeferredFFromCoroF:
yield self._assert_result(c_f)
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_coroutine_asyncio(self):
async def c_f() -> int:
await asyncio.sleep(0.01)
@ -337,7 +336,7 @@ class TestDeferredFFromCoroF:
yield self._assert_result(c_f)
@pytest.mark.only_asyncio
@inlineCallbacks
@inline_callbacks_test
def test_future(self):
def c_f() -> Future[int]:
f: Future[int] = Future()

6
tests/test_utils_python.py
View File

@ -20,7 +20,7 @@ from scrapy.utils.python import (
to_unicode,
without_none_values,
)
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
@ -64,7 +64,7 @@ class TestMutableAsyncChain:
for i in range(5, 7):
yield i
@deferred_f_from_coro_f
@coroutine_test
async def test_mutableasyncchain(self):
m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
m.extend(self.g2())
@ -74,7 +74,7 @@ class TestMutableAsyncChain:
results = await collect_asyncgen(m)
assert results == list(range(1, 10))
@deferred_f_from_coro_f
@coroutine_test
async def test_mutableasyncchain_exc(self):
m = MutableAsyncChain(self.g1())
m.extend(self.g4())

4
tests/test_utils_reactor.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.reactor import (
is_asyncio_reactor_installed,
set_asyncio_event_loop,
)
from tests.utils.decorators import deferred_f_from_coro_f
from tests.utils.decorators import coroutine_test
class TestAsyncio:
@ -31,7 +31,7 @@ class TestAsyncio:
@pytest.mark.requires_reactor
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@coroutine_test
async def test_set_asyncio_event_loop(self):
install_reactor(_asyncio_reactor_path)
assert set_asyncio_event_loop(None) is asyncio.get_running_loop()

4
tests/test_utils_signal.py
View File

@ -14,14 +14,14 @@ from scrapy.utils.signal import (
send_catch_log_deferred,
)
from scrapy.utils.test import get_from_asyncio_queue
from tests.utils.decorators import inlineCallbacks
from tests.utils.decorators import inline_callbacks_test
class TestSendCatchLog:
# whether the function being tested returns exceptions or failures
returns_exceptions: bool = False
@inlineCallbacks
@inline_callbacks_test
def test_send_catch_log(self):
test_signal = object()
handlers_called = set()

11
tests/utils/decorators.py
View File

@ -4,8 +4,7 @@ from functools import wraps
from typing import TYPE_CHECKING, Any, ParamSpec
import pytest
from twisted.internet.defer import Deferred
from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
from twisted.internet.defer import Deferred, inlineCallbacks
from scrapy.utils.defer import deferred_from_coro, deferred_to_future
from scrapy.utils.reactor import is_reactor_installed
@ -17,7 +16,7 @@ if TYPE_CHECKING:
_P = ParamSpec("_P")
def inlineCallbacks(
def inline_callbacks_test(
f: Callable[_P, Generator[Deferred[Any], Any, None]],
) -> Callable[_P, Awaitable[None]]:
"""Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style.
@ -34,12 +33,12 @@ def inlineCallbacks(
@pytest.mark.asyncio
@wraps(f)
async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None:
await deferred_to_future(inlineCallbacks_orig(f)(*args, **kwargs))
await deferred_to_future(inlineCallbacks(f)(*args, **kwargs))
return wrapper_coro
@wraps(f)
@inlineCallbacks_orig
@inlineCallbacks
def wrapper_dfd(
*args: _P.args, **kwargs: _P.kwargs
) -> Generator[Deferred[Any], Any, None]:
@ -48,7 +47,7 @@ def inlineCallbacks(
return wrapper_dfd
def deferred_f_from_coro_f(
def coroutine_test(
coro_f: Callable[_P, Awaitable[None]],
) -> Callable[_P, Awaitable[None]]:
"""Mark a test function that returns a coroutine.