diff --git a/tests/test_addons.py b/tests/test_addons.py index 675574d48..5c26e4651 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -7,7 +7,7 @@ from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler, get_reactor_settings -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class SimpleAddon: @@ -183,7 +183,7 @@ class TestAddonManager: extra={"crawler": crawler}, ) - @inlineCallbacks + @inline_callbacks_test def test_enable_addon_in_spider(self): class MySpider(Spider): name = "myspider" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 20c311c5a..a6fd6dd19 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -9,7 +9,7 @@ from tests.spiders import ( MaxItemsAndRequestsSpider, SlowSpider, ) -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test @pytest.mark.requires_http_handler @@ -23,7 +23,7 @@ class TestCloseSpider: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) @@ -33,7 +33,7 @@ class TestCloseSpider: itemcount = crawler.stats.get_value("item_scraped_count") assert itemcount >= close_on - @inlineCallbacks + @inline_callbacks_test def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) @@ -43,7 +43,7 @@ class TestCloseSpider: pagecount = crawler.stats.get_value("response_received_count") assert pagecount >= close_on - @inlineCallbacks + @inline_callbacks_test def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 @@ -63,7 +63,7 @@ class TestCloseSpider: itemcount = crawler.stats.get_value("item_scraped_count") assert pagecount <= close_on + itemcount - @inlineCallbacks + @inline_callbacks_test def test_closespider_pagecount_no_item_with_pagecount(self): close_on_pagecount_no_item = 5 close_on_pagecount = 20 @@ -80,7 +80,7 @@ class TestCloseSpider: pagecount = crawler.stats.get_value("response_received_count") assert pagecount < close_on_pagecount - @inlineCallbacks + @inline_callbacks_test def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) @@ -92,7 +92,7 @@ class TestCloseSpider: assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on - @inlineCallbacks + @inline_callbacks_test def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) @@ -102,7 +102,7 @@ class TestCloseSpider: total_seconds = crawler.stats.get_value("elapsed_time_seconds") assert total_seconds >= close_on - @inlineCallbacks + @inline_callbacks_test def test_closespider_timeout_no_item(self): timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 9dface8aa..f1678f912 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -18,7 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class DemoItem(Item): @@ -502,7 +502,7 @@ class TestContractsManager: assert self.results.errors @pytest.mark.requires_http_handler - @inlineCallbacks + @inline_callbacks_test def test_same_url(self): class TestSameUrlSpider(Spider): name = "test_same_url" diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index c479e66c9..2a098f9f3 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -25,7 +25,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http_resources import PayloadResource from tests.mockserver.utils import ssl_context_factory -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred @@ -97,7 +97,7 @@ class TestContextFactoryBase: class TestContextFactory(TestContextFactoryBase): - @deferred_f_from_coro_f + @coroutine_test async def testPayload(self, server_url: str) -> None: s = "0123456789" * 10 crawler = get_crawler() @@ -135,7 +135,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): ) assert body == to_bytes(s) - @deferred_f_from_coro_f + @coroutine_test async def test_setting_default(self, server_url: str) -> None: crawler = get_crawler() settings = Settings() @@ -155,7 +155,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): with pytest.raises(KeyError): load_context_factory_from_settings(settings, crawler) - @deferred_f_from_coro_f + @coroutine_test async def test_setting_explicit(self, server_url: str) -> None: crawler = get_crawler() settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) @@ -163,7 +163,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) - @deferred_f_from_coro_f + @coroutine_test async def test_direct_from_crawler(self, server_url: str) -> None: # the setting is ignored crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) @@ -171,14 +171,14 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD await self._assert_factory_works(server_url, client_context_factory) - @deferred_f_from_coro_f + @coroutine_test async def test_direct_init(self, server_url: str) -> None: client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) -@deferred_f_from_coro_f +@coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): def fetch(self, request, spider): # pylint: disable=signature-differs diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py index 249daf943..f4ef5ec5d 100644 --- a/tests/test_core_scraper.py +++ b/tests/test_core_scraper.py @@ -6,14 +6,14 @@ import pytest from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from tests.mockserver.http import MockServer @pytest.mark.requires_http_handler -@deferred_f_from_coro_f +@coroutine_test async def test_scraper_exception( mockserver: MockServer, caplog: pytest.LogCaptureFixture, diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 61a4d387f..b206cfffa 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -55,7 +55,7 @@ from tests.spiders import ( StartGoodAndBadOutput, StartItemSpider, ) -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector @@ -74,17 +74,17 @@ class TestCrawl: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @deferred_f_from_coro_f + @coroutine_test async def test_fixed_delay(self): await self._test_delay(total=3, delay=0.2) - @deferred_f_from_coro_f + @coroutine_test async def test_randomized_delay(self): await self._test_delay(total=3, delay=0.1, randomize=True) @@ -122,7 +122,7 @@ class TestCrawl: average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @inlineCallbacks + @inline_callbacks_test def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -130,7 +130,7 @@ class TestCrawl: assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @inlineCallbacks + @inline_callbacks_test def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -147,7 +147,7 @@ class TestCrawl: assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @inlineCallbacks + @inline_callbacks_test def test_retry_503(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -156,7 +156,7 @@ class TestCrawl: ) self._assert_retried(log) - @inlineCallbacks + @inline_callbacks_test def test_retry_conn_failed(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -165,7 +165,7 @@ class TestCrawl: ) self._assert_retried(log) - @inlineCallbacks + @inline_callbacks_test def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") @@ -177,7 +177,7 @@ class TestCrawl: ) self._assert_retried(log) - @inlineCallbacks + @inline_callbacks_test def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -188,7 +188,7 @@ class TestCrawl: assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inlineCallbacks + @inline_callbacks_test def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -199,7 +199,7 @@ class TestCrawl: assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inlineCallbacks + @inline_callbacks_test def test_start_items(self): items = [] @@ -214,7 +214,7 @@ class TestCrawl: assert len(log.records) == 0 assert items == [{"name": "test item"}] - @inlineCallbacks + @inline_callbacks_test def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting @@ -235,7 +235,7 @@ class TestCrawl: assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @inlineCallbacks + @inline_callbacks_test def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) @@ -253,7 +253,7 @@ class TestCrawl: ) assert crawler.spider.visited == 3 - @inlineCallbacks + @inline_callbacks_test def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" @@ -285,7 +285,7 @@ with multiples lines ) assert str(log).count("Got response 200") == 1 - @inlineCallbacks + @inline_callbacks_test def test_retry_conn_lost(self): # connection lost after receiving data crawler = get_crawler(SimpleSpider) @@ -295,7 +295,7 @@ with multiples lines ) self._assert_retried(log) - @inlineCallbacks + @inline_callbacks_test def test_retry_conn_aborted(self): # connection lost before receiving data crawler = get_crawler(SimpleSpider) @@ -309,7 +309,7 @@ with multiples lines assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @inlineCallbacks + @inline_callbacks_test def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) @@ -337,7 +337,7 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @inlineCallbacks + @inline_callbacks_test def test_engine_status(self): est = [] @@ -353,7 +353,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @inlineCallbacks + @inline_callbacks_test def test_format_engine_status(self): est = [] @@ -376,7 +376,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @inlineCallbacks + @inline_callbacks_test def test_open_spider_error_on_faulty_pipeline(self): settings = { "ITEM_PIPELINES": { @@ -390,7 +390,7 @@ with multiples lines ) assert not crawler.crawling - @inlineCallbacks + @inline_callbacks_test def test_crawlerrunner_accepts_crawler(self): crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() @@ -402,7 +402,7 @@ with multiples lines ) assert "Got response 200" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture): runner = CrawlerRunner(get_reactor_settings()) runner.crawl( @@ -422,7 +422,7 @@ with multiples lines self._assert_retried(caplog.text) assert "Got response 200" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(SimpleSpider) await maybe_deferred_to_future(crawler.crawl("foo://bar")) @@ -461,7 +461,7 @@ class TestCrawlSpider: assert crawler.stats return log, items, crawler.stats - @inlineCallbacks + @inline_callbacks_test def test_crawlspider_with_parse(self): crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: @@ -471,7 +471,7 @@ class TestCrawlSpider: assert "[parse] status 201 (foo: None)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_crawlspider_with_async_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncCallback) with LogCapture() as log: @@ -481,7 +481,7 @@ class TestCrawlSpider: assert "[parse_async] status 201 (foo: None)" in str(log) assert "[parse_async] status 202 (foo: bar)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_crawlspider_with_async_generator_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) with LogCapture() as log: @@ -491,7 +491,7 @@ class TestCrawlSpider: assert "[parse_async_gen] status 201 (foo: None)" in str(log) assert "[parse_async_gen] status 202 (foo: bar)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: @@ -504,7 +504,7 @@ class TestCrawlSpider: assert "[errback] status 500" in str(log) assert "[errback] status 501" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_crawlspider_process_request_cb_kwargs(self): crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) with LogCapture() as log: @@ -514,7 +514,7 @@ class TestCrawlSpider: assert "[parse] status 201 (foo: process_request)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: @@ -524,7 +524,7 @@ class TestCrawlSpider: assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_async_def_asyncio_parse(self): crawler = get_crawler( AsyncDefAsyncioSpider, @@ -539,7 +539,7 @@ class TestCrawlSpider: assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncio_parse_items_list(self): log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) @@ -547,7 +547,7 @@ class TestCrawlSpider: assert {"id": 2} in items @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -564,7 +564,7 @@ class TestCrawlSpider: assert {"foo": 42} in items @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncgen_parse(self): log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) @@ -572,7 +572,7 @@ class TestCrawlSpider: assert itemcount == 1 @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncgen_parse_loop(self): log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) @@ -582,7 +582,7 @@ class TestCrawlSpider: assert {"foo": i} in items @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncgen_parse_exc(self): log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) @@ -594,7 +594,7 @@ class TestCrawlSpider: assert {"foo": i} in items @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncgen_parse_complex(self): _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") @@ -606,30 +606,30 @@ class TestCrawlSpider: assert {"index2": i} in items @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_asyncio_parse_reqs_list(self): log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_deferred_direct(self): _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_deferred_wrapped(self): _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @deferred_f_from_coro_f + @coroutine_test async def test_async_def_deferred_maybe_wrapped(self): _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] - @inlineCallbacks + @inline_callbacks_test def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) @@ -649,7 +649,7 @@ class TestCrawlSpider: ), ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_response_ssl_certificate( self, mockserver: MockServer, url: str ) -> None: @@ -675,7 +675,7 @@ class TestCrawlSpider: ), ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None: crawler = get_crawler(SingleRequestSpider) url = mockserver.url(url) @@ -686,7 +686,7 @@ class TestCrawlSpider: assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @inlineCallbacks + @inline_callbacks_test def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -700,7 +700,7 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inlineCallbacks + @inline_callbacks_test def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -716,7 +716,7 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inlineCallbacks + @inline_callbacks_test def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -726,7 +726,7 @@ class TestCrawlSpider: "headers_received" ) - @inlineCallbacks + @inline_callbacks_test def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -738,7 +738,7 @@ class TestCrawlSpider: "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @inlineCallbacks + @inline_callbacks_test def test_spider_callback_deferred_deprecated(self): def cb(response: Response) -> Any: return succeed(None) @@ -750,7 +750,7 @@ class TestCrawlSpider: ): yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback(self): failures = [] @@ -767,7 +767,7 @@ class TestCrawlSpider: assert "HTTP status code is not handled or not allowed" in str(log) assert "Spider error processing" not in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_silence(self): failures = [] @@ -783,7 +783,7 @@ class TestCrawlSpider: assert "HTTP status code is not handled or not allowed" not in str(log) assert "Spider error processing" not in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -795,7 +795,7 @@ class TestCrawlSpider: ) assert "Spider error processing" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -809,7 +809,7 @@ class TestCrawlSpider: assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -823,7 +823,7 @@ class TestCrawlSpider: assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_downloader_error(self): failures = [] @@ -840,7 +840,7 @@ class TestCrawlSpider: assert "Error downloading" in str(log) assert "Spider error processing" not in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -853,7 +853,7 @@ class TestCrawlSpider: assert "Error downloading" in str(log) assert "Spider error processing" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_downloader_error_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -867,7 +867,7 @@ class TestCrawlSpider: assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_downloader_error_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -881,7 +881,7 @@ class TestCrawlSpider: assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @inlineCallbacks + @inline_callbacks_test def test_spider_errback_deferred_deprecated(self): def eb(failure: Failure) -> Any: return succeed(None) @@ -895,7 +895,7 @@ class TestCrawlSpider: seed=self.mockserver.url("/status?n=400"), errback_func=eb ) - @inlineCallbacks + @inline_callbacks_test def test_raise_closespider(self): def cb(response): raise CloseSpider @@ -906,7 +906,7 @@ class TestCrawlSpider: assert "Closing spider (cancelled)" in str(log) assert "Spider error processing" not in str(log) - @inlineCallbacks + @inline_callbacks_test def test_raise_closespider_reason(self): def cb(response): raise CloseSpider("my_reason") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8de38114c..ba33d0bbe 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -41,7 +41,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver.http import MockServer from tests.utils import get_script_run_env -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test BASE_SETTINGS: dict[str, Any] = {} @@ -101,21 +101,21 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(ValueError, match="spidercls argument must be a class"): Crawler(DefaultSpider()) - @inlineCallbacks + @inline_callbacks_test def test_crawler_crawl_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() - @deferred_f_from_coro_f + @coroutine_test async def test_crawler_crawl_async_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) await crawler.crawl_async() with pytest.raises(RuntimeError, match="more than once on the same instance"): await crawler.crawl_async() - @inlineCallbacks + @inline_callbacks_test def test_crawler_crawl_twice_parallel_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) d1 = crawler.crawl() @@ -125,7 +125,7 @@ class TestCrawler(TestBaseCrawler): yield d2 @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_crawler_crawl_async_twice_parallel_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) t1 = asyncio.create_task(crawler.crawl_async()) @@ -172,7 +172,7 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(ChildAddon) assert addon is None - @inlineCallbacks + @inline_callbacks_test def test_get_downloader_middleware(self): class ParentDownloaderMiddleware: pass @@ -235,7 +235,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): crawler.get_downloader_middleware(DefaultSpider) - @inlineCallbacks + @inline_callbacks_test def test_get_downloader_middleware_no_engine(self): class MySpider(Spider): name = "myspider" @@ -252,7 +252,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): yield crawler.crawl() - @inlineCallbacks + @inline_callbacks_test def test_get_extension(self): class ParentExtension: pass @@ -315,7 +315,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): crawler.get_extension(DefaultSpider) - @inlineCallbacks + @inline_callbacks_test def test_get_extension_no_engine(self): class MySpider(Spider): name = "myspider" @@ -332,7 +332,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): yield crawler.crawl() - @inlineCallbacks + @inline_callbacks_test def test_get_item_pipeline(self): class ParentItemPipeline: pass @@ -395,7 +395,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): crawler.get_item_pipeline(DefaultSpider) - @inlineCallbacks + @inline_callbacks_test def test_get_item_pipeline_no_engine(self): class MySpider(Spider): name = "myspider" @@ -412,7 +412,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): yield crawler.crawl() - @inlineCallbacks + @inline_callbacks_test def test_get_spider_middleware(self): class ParentSpiderMiddleware: pass @@ -475,7 +475,7 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError): crawler.get_spider_middleware(DefaultSpider) - @inlineCallbacks + @inline_callbacks_test def test_get_spider_middleware_no_engine(self): class MySpider(Spider): name = "myspider" @@ -516,7 +516,7 @@ class TestCrawlerLogging: get_crawler(MySpider) assert get_scrapy_root_handler() is None - @deferred_f_from_coro_f + @coroutine_test async def test_spider_custom_settings_log_level(self, tmp_path): log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -686,20 +686,20 @@ class TestCrawlerRunnerHasSpider: def _crawl(runner, spider): return runner.crawl(spider) - @inlineCallbacks + @inline_callbacks_test def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inlineCallbacks + @inline_callbacks_test def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield self._crawl(runner, NoRequestsSpider) yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inlineCallbacks + @inline_callbacks_test def test_crawler_runner_bootstrap_failed(self): runner = self._runner() @@ -712,7 +712,7 @@ class TestCrawlerRunnerHasSpider: assert runner.bootstrap_failed - @inlineCallbacks + @inline_callbacks_test def test_crawler_runner_bootstrap_failed_for_several(self): runner = self._runner() @@ -727,7 +727,7 @@ class TestCrawlerRunnerHasSpider: assert runner.bootstrap_failed - @inlineCallbacks + @inline_callbacks_test def test_crawler_runner_asyncio_enabled_true( self, reactor_pytest: str ) -> Generator[Deferred[Any], Any, None]: @@ -959,7 +959,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.expect_exact("Spider closed (shutdown)") p.wait() - @inlineCallbacks + @inline_callbacks_test def test_shutdown_forced(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK args = self.get_script_args("sleeping.py", "10") @@ -1297,7 +1297,7 @@ def test_log_scrapy_info(settings, items, caplog): assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string) -@deferred_f_from_coro_f +@coroutine_test async def test_deprecated_crawler_stop() -> None: crawler = get_crawler(DefaultSpider) d = crawler.crawl() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7844415f5..4589e667f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -21,7 +21,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.boto import is_botocore_available from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test class DummyDH: @@ -126,7 +126,7 @@ class TestFile: os.close(self.fd) Path(self.tmpname).unlink() - @deferred_f_from_coro_f + @coroutine_test async def test_download(self): request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") @@ -136,7 +136,7 @@ class TestFile: assert response.body == b"0123456789" assert response.protocol is None - @deferred_f_from_coro_f + @coroutine_test async def test_non_existent(self): request = Request(path_to_file_uri(mkdtemp())) # the specific exception differs between platforms @@ -163,7 +163,7 @@ class TestS3Anon: self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.download_request = self.s3reqh.download_request - @deferred_f_from_coro_f + @coroutine_test async def test_anon_request(self): req = Request("s3://aws-publicdatasets/") httpreq = await self.download_request(req) @@ -205,7 +205,7 @@ class TestS3: mock_formatdate.return_value = date yield - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing1(self): # gets an object from the johnsmith bucket. date = "Tue, 27 Mar 2007 19:36:42 +0000" @@ -217,7 +217,7 @@ class TestS3: == b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing2(self): # puts an object into the johnsmith bucket. date = "Tue, 27 Mar 2007 21:15:45 +0000" @@ -237,7 +237,7 @@ class TestS3: == b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing3(self): # lists the content of the johnsmith bucket. date = "Tue, 27 Mar 2007 19:42:41 +0000" @@ -256,7 +256,7 @@ class TestS3: == b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. date = "Tue, 27 Mar 2007 19:44:46 +0000" @@ -268,7 +268,7 @@ class TestS3: == b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = "Tue, 27 Mar 2007 21:06:08 +0000" @@ -297,7 +297,7 @@ class TestS3: == b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_signing7(self): # ensure that spaces are quoted properly before signing date = "Tue, 27 Mar 2007 19:42:41 +0000" @@ -327,7 +327,7 @@ class TestDataURI: download_handler = build_from_crawler(DataURIDownloadHandler, crawler) self.download_request = download_handler.download_request - @deferred_f_from_coro_f + @coroutine_test async def test_response_attrs(self): uri = "data:,A%20brief%20note" request = Request(uri) @@ -335,7 +335,7 @@ class TestDataURI: assert response.url == uri assert not response.headers - @deferred_f_from_coro_f + @coroutine_test async def test_default_mediatype_encoding(self): request = Request("data:,A%20brief%20note") response = await self.download_request(request) @@ -343,7 +343,7 @@ class TestDataURI: assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert response.encoding == "US-ASCII" - @deferred_f_from_coro_f + @coroutine_test async def test_default_mediatype(self): request = Request("data:;charset=iso-8859-7,%be%d3%be") response = await self.download_request(request) @@ -351,7 +351,7 @@ class TestDataURI: assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert response.encoding == "iso-8859-7" - @deferred_f_from_coro_f + @coroutine_test async def test_text_charset(self): request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") response = await self.download_request(request) @@ -359,7 +359,7 @@ class TestDataURI: assert response.body == b"\xbe\xd3\xbe" assert response.encoding == "iso-8859-7" - @deferred_f_from_coro_f + @coroutine_test async def test_mediatype_parameters(self): request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" @@ -371,13 +371,13 @@ class TestDataURI: assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert response.encoding == "utf-8" - @deferred_f_from_coro_f + @coroutine_test async def test_base64(self): request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") response = await self.download_request(request) assert response.text == "Hello, world." - @deferred_f_from_coro_f + @coroutine_test async def test_protocol(self): request = Request("data:,") response = await self.download_request(request) diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 4d4b4ecaa..ff76c88fe 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -34,7 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer from tests.spiders import SingleRequestSpider -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator @@ -63,21 +63,21 @@ class TestHttpBase(ABC): finally: await dh.close() - @deferred_f_from_coro_f + @coroutine_test async def test_unsupported_scheme(self) -> None: request = Request("ftp://unsupported.scheme") async with self.get_dh() as download_handler: with pytest.raises(UnsupportedURLSchemeError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.body == b"Works" - @deferred_f_from_coro_f + @coroutine_test async def test_download_head(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/text", is_secure=self.is_secure), method="HEAD" @@ -94,7 +94,7 @@ class TestHttpBase(ABC): if http_status.value == 200 or http_status.value // 100 in (4, 5) ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_download_has_correct_http_status_code( self, mockserver: MockServer, http_status: HTTPStatus ) -> None: @@ -105,7 +105,7 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert response.status == http_status.value - @deferred_f_from_coro_f + @coroutine_test async def test_server_receives_correct_request_headers( self, mockserver: MockServer ) -> None: @@ -132,7 +132,7 @@ class TestHttpBase(ABC): assert header_name in body["headers"] assert body["headers"][header_name] == [header_value] - @deferred_f_from_coro_f + @coroutine_test async def test_request_header_none(self, mockserver: MockServer) -> None: """Adding a header with None as the value should not send that header.""" request_headers = { @@ -158,7 +158,7 @@ class TestHttpBase(ABC): [("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")], ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_request_header_duplicate( self, mockserver: MockServer, request_headers: Any ) -> None: @@ -174,7 +174,7 @@ class TestHttpBase(ABC): assert "headers" in body assert body["headers"]["X-Custom-Header"] == ["foo", "bar"] - @deferred_f_from_coro_f + @coroutine_test async def test_server_receives_correct_request_body( self, mockserver: MockServer ) -> None: @@ -191,7 +191,7 @@ class TestHttpBase(ABC): body = json.loads(response.body.decode("utf-8")) assert json.loads(body["body"]) == request_body - @deferred_f_from_coro_f + @coroutine_test async def test_download_has_correct_response_headers( self, mockserver: MockServer ) -> None: @@ -230,14 +230,14 @@ class TestHttpBase(ABC): header_value, encoding="utf-8" ) - @deferred_f_from_coro_f + @coroutine_test async def test_redirect_status(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/redirect", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.status == 302 - @deferred_f_from_coro_f + @coroutine_test async def test_redirect_status_head(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD" @@ -246,7 +246,7 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert response.status == 302 - @deferred_f_from_coro_f + @coroutine_test async def test_timeout_download_from_spider_nodata_rcvd( self, mockserver: MockServer, reactor_pytest: str ) -> None: @@ -264,7 +264,7 @@ class TestHttpBase(ABC): with pytest.raises(DownloadTimeoutError): await maybe_deferred_to_future(d) - @deferred_f_from_coro_f + @coroutine_test async def test_timeout_download_from_spider_server_hangs( self, mockserver: MockServer, @@ -286,7 +286,7 @@ class TestHttpBase(ABC): await maybe_deferred_to_future(d) @pytest.mark.parametrize("send_header", [True, False]) - @deferred_f_from_coro_f + @coroutine_test async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}" request = Request( @@ -301,7 +301,7 @@ class TestHttpBase(ABC): else: assert not request.headers - @deferred_f_from_coro_f + @coroutine_test async def test_content_length_zero_bodyless_post_request_headers( self, mockserver: MockServer ) -> None: @@ -322,7 +322,7 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert response.body == b"0" - @deferred_f_from_coro_f + @coroutine_test async def test_content_length_zero_bodyless_post_only_one( self, mockserver: MockServer ) -> None: @@ -336,7 +336,7 @@ class TestHttpBase(ABC): assert len(contentlengths) == 1 assert contentlengths == [b"0"] - @deferred_f_from_coro_f + @coroutine_test async def test_payload(self, mockserver: MockServer) -> None: body = b"1" * 100 # PayloadResource requires body length to be 100 request = Request( @@ -348,7 +348,7 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert response.body == body - @deferred_f_from_coro_f + @coroutine_test async def test_response_header_content_length(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/text", is_secure=self.is_secure), method="GET" @@ -364,7 +364,7 @@ class TestHttpBase(ABC): ("foo", b"\n.", HtmlResponse), ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_response_class( self, filename: str, @@ -379,14 +379,14 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - @deferred_f_from_coro_f + @coroutine_test async def test_get_duplicate_header(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - @deferred_f_from_coro_f + @coroutine_test async def test_download_is_not_automatically_gzip_decoded( self, mockserver: MockServer ) -> None: @@ -420,7 +420,7 @@ class TestHttpBase(ABC): expected_decoding = bytes(data, encoding="utf-8") assert gzip.decompress(response.body) == expected_decoding - @deferred_f_from_coro_f + @coroutine_test async def test_no_cookie_processing_or_persistence( self, mockserver: MockServer ) -> None: @@ -451,14 +451,14 @@ class TestHttpBase(ABC): class TestHttp11Base(TestHttpBase): """HTTP 1.1 test case""" - @deferred_f_from_coro_f + @coroutine_test async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.body == b"Works" - @deferred_f_from_coro_f + @coroutine_test async def test_response_class_choosing_request( self, mockserver: MockServer ) -> None: @@ -473,7 +473,7 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_maxsize(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) @@ -487,7 +487,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(DownloadCancelledError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_maxsize_very_large_file( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture ) -> None: @@ -498,7 +498,7 @@ class TestHttp11Base(TestHttpBase): assert "larger than download max size" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None: meta = {"download_maxsize": 2} request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta) @@ -506,7 +506,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(DownloadCancelledError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_small_maxsize_via_setting( self, mockserver: MockServer ) -> None: @@ -515,7 +515,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(DownloadCancelledError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_large_maxsize_via_setting( self, mockserver: MockServer ) -> None: @@ -524,7 +524,7 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert response.body == b"Works" - @deferred_f_from_coro_f + @coroutine_test async def test_download_chunked_content(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/chunked", is_secure=self.is_secure)) async with self.get_dh() as download_handler: @@ -532,7 +532,7 @@ class TestHttp11Base(TestHttpBase): assert response.body == b"chunked content\n" @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) - @deferred_f_from_coro_f + @coroutine_test async def test_download_cause_data_loss( self, url: str, mockserver: MockServer ) -> None: @@ -541,7 +541,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(ResponseDataLossError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_cause_data_loss_double_warning( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: @@ -557,7 +557,7 @@ class TestHttp11Base(TestHttpBase): assert "Got data loss" not in caplog.text @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) - @deferred_f_from_coro_f + @coroutine_test async def test_download_allow_data_loss( self, url: str, mockserver: MockServer ) -> None: @@ -570,7 +570,7 @@ class TestHttp11Base(TestHttpBase): assert response.flags == ["dataloss"] @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) - @deferred_f_from_coro_f + @coroutine_test async def test_download_allow_data_loss_via_setting( self, url: str, mockserver: MockServer ) -> None: @@ -581,7 +581,7 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert response.flags == ["dataloss"] - @deferred_f_from_coro_f + @coroutine_test async def test_download_conn_failed(self) -> None: # copy of TestCrawl.test_retry_conn_failed() scheme = "https" if self.is_secure else "http" @@ -590,7 +590,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(DownloadConnectionRefusedError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_conn_lost(self, mockserver: MockServer) -> None: # copy of TestCrawl.test_retry_conn_lost() request = Request(mockserver.url("/drop?abort=0", is_secure=self.is_secure)) @@ -598,7 +598,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(ResponseDataLossError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_conn_aborted(self, mockserver: MockServer) -> None: # copy of TestCrawl.test_retry_conn_aborted() request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure)) @@ -609,7 +609,7 @@ class TestHttp11Base(TestHttpBase): @pytest.mark.skipif( NON_EXISTING_RESOLVABLE, reason="Non-existing hosts are resolvable" ) - @deferred_f_from_coro_f + @coroutine_test async def test_download_dns_error(self) -> None: # copy of TestCrawl.test_retry_dns_error() scheme = "https" if self.is_secure else "http" @@ -618,7 +618,7 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(CannotResolveHostError): await download_handler.download_request(request) - @deferred_f_from_coro_f + @coroutine_test async def test_protocol(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/host", is_secure=self.is_secure), method="GET" @@ -642,7 +642,7 @@ class TestHttps11Base(TestHttp11Base): # (not just Scrapy) hang on /drop?abort=0. pytest.skip("Unable to test on HTTPS") - @deferred_f_from_coro_f + @coroutine_test async def test_tls_logging( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture ) -> None: @@ -695,7 +695,7 @@ class TestSimpleHttpsBase(ABC): finally: await dh.close() - @deferred_f_from_coro_f + @coroutine_test async def test_download(self, url: str) -> None: request = Request(url) async with self.get_dh() as download_handler: @@ -738,7 +738,7 @@ class TestHttpWithCrawlerBase(ABC): is_secure = False - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_content_length(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid @@ -755,7 +755,7 @@ class TestHttpWithCrawlerBase(ABC): failure = crawler.spider.meta["failure"] # type: ignore[attr-defined] assert isinstance(failure.value, DownloadCancelledError) - @deferred_f_from_coro_f + @coroutine_test async def test_download(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider, self.settings_dict) await maybe_deferred_to_future( @@ -769,7 +769,7 @@ class TestHttpWithCrawlerBase(ABC): reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] assert reason == "finished" - @deferred_f_from_coro_f + @coroutine_test async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: if not self.is_secure: pytest.skip("Only applies to HTTPS") @@ -784,7 +784,7 @@ class TestHttpWithCrawlerBase(ABC): assert cert.getSubject().commonName == b"localhost" assert cert.getIssuer().commonName == b"localhost" - @deferred_f_from_coro_f + @coroutine_test async def test_response_ip_address(self, mockserver: MockServer) -> None: # copy of TestCrawl.test_response_ip_address() crawler = get_crawler(SingleRequestSpider, self.settings_dict) @@ -821,7 +821,7 @@ class TestHttpProxyBase(ABC): finally: await dh.close() - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_proxy( self, proxy_mockserver: ProxyEchoMockServer ) -> None: @@ -833,7 +833,7 @@ class TestHttpProxyBase(ABC): assert response.url == request.url assert response.body == self.expected_http_proxy_request_body - @deferred_f_from_coro_f + @coroutine_test async def test_download_without_proxy( self, proxy_mockserver: ProxyEchoMockServer ) -> None: @@ -846,7 +846,7 @@ class TestHttpProxyBase(ABC): assert response.url == request.url assert response.body == b"/path/to/resource" - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_proxy_https_timeout( self, proxy_mockserver: ProxyEchoMockServer ) -> None: @@ -860,7 +860,7 @@ class TestHttpProxyBase(ABC): await download_handler.download_request(request) assert domain in str(exc_info.value) - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_proxy_without_http_scheme( self, proxy_mockserver: ProxyEchoMockServer ) -> None: diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 10ac77aa0..d321c9047 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -16,7 +16,7 @@ from scrapy.spiders import Spider from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -60,7 +60,7 @@ class TestManagerBase: class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - @deferred_f_from_coro_f + @coroutine_test async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) @@ -68,7 +68,7 @@ class TestDefaults(TestManagerBase): ret = await self._download(mwman, req, resp) assert isinstance(ret, Response), "Non-response returned" - @deferred_f_from_coro_f + @coroutine_test async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -101,7 +101,7 @@ class TestDefaults(TestManagerBase): "Not redirected to location header" ) - @deferred_f_from_coro_f + @coroutine_test async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" @@ -124,7 +124,7 @@ class TestDefaults(TestManagerBase): class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - @deferred_f_from_coro_f + @coroutine_test async def test_download_func_not_called(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") @@ -144,7 +144,7 @@ class TestResponseFromProcessRequest(TestManagerBase): class TestResponseFromProcessException(TestManagerBase): """Tests middleware returning a response from process_exception.""" - @deferred_f_from_coro_f + @coroutine_test async def test_process_response_called(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") @@ -173,7 +173,7 @@ class TestResponseFromProcessException(TestManagerBase): class TestInvalidOutput(TestManagerBase): - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_request(self): """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") @@ -187,7 +187,7 @@ class TestInvalidOutput(TestManagerBase): with pytest.raises(_InvalidOutput): await self._download(mwman, req) - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_response(self): """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") @@ -201,7 +201,7 @@ class TestInvalidOutput(TestManagerBase): with pytest.raises(_InvalidOutput): await self._download(mwman, req) - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_exception(self): """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") @@ -222,7 +222,7 @@ class TestInvalidOutput(TestManagerBase): class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds (deprecated) should work""" - @deferred_f_from_coro_f + @coroutine_test async def test_deferred(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") @@ -252,7 +252,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - @deferred_f_from_coro_f + @coroutine_test async def test_asyncdef(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") @@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase): assert not download_func.called @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_asyncdef_asyncio(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") @@ -289,7 +289,7 @@ class TestMiddlewareUsingCoro(TestManagerBase): class TestDownloadDeprecated(TestManagerBase): - @deferred_f_from_coro_f + @coroutine_test async def test_mwman_download(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) @@ -309,7 +309,7 @@ class TestDownloadDeprecated(TestManagerBase): class TestDeprecatedSpiderArg(TestManagerBase): - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg(self): req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9b4b2750b..15c68779d 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -16,7 +16,7 @@ from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.test_robotstxt_interface import rerp_available -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -60,7 +60,7 @@ Disallow: /some/randome/page.html crawler.engine.download_async.side_effect = return_response return crawler - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) @@ -74,7 +74,7 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) d1 = deferred_from_coro( @@ -86,20 +86,20 @@ Disallow: /some/randome/page.html await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_multiple_reqs_asyncio(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) c1 = middleware.process_request(Request("http://site.local/allowed1")) c2 = middleware.process_request(Request("http://site.local/allowed2")) await asyncio.gather(c1, c2) - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_ready_parser(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {"dont_obey_robotstxt": True} @@ -128,7 +128,7 @@ Disallow: /some/randome/page.html crawler.engine.download_async.side_effect = return_response return crawler - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) @@ -150,7 +150,7 @@ Disallow: /some/randome/page.html crawler.engine.download_async.side_effect = return_response return crawler - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) @@ -158,7 +158,7 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None: self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") @@ -174,7 +174,7 @@ Disallow: /some/randome/page.html await middleware.process_request(Request("http://site.local")) assert "Robotstxt address not found" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_immediate_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") @@ -187,7 +187,7 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) await self.assertNotIgnored(Request("http://site.local"), middleware) - @deferred_f_from_coro_f + @coroutine_test async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) @@ -216,7 +216,7 @@ Disallow: /some/randome/page.html middleware.process_request_2(rp, Request("http://site.local/allowed")) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") - @deferred_f_from_coro_f + @coroutine_test async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) middleware.process_request_2 = mock.MagicMock() diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 7d2001e47..5ddb1cdfa 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MetaSpider -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test class DownloaderSlotsSettingsTestSpider(MetaSpider): @@ -68,7 +68,7 @@ class TestCrawl: self.runner = CrawlerRunner() @pytest.mark.requires_http_handler - @inlineCallbacks + @inline_callbacks_test def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -137,7 +137,7 @@ def test_get_slot_deprecated_spider_arg(): "scrapy.pqueues.DownloaderAwarePriorityQueue", ], ) -@deferred_f_from_coro_f +@coroutine_test async def test_none_slot_with_priority_queue( mockserver: MockServer, priority_queue_class: str ) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index 939ea0f6f..c77e3c5c9 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -35,7 +35,7 @@ from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import get_testdata -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from scrapy.core.scheduler import Scheduler @@ -374,7 +374,7 @@ class TestEngineBase: class TestEngine(TestEngineBase): @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler(self, mockserver: MockServer) -> None: for spider in ( MySpider, @@ -392,7 +392,7 @@ class TestEngine(TestEngineBase): self._assert_bytes_received(run) @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler_dupefilter(self, mockserver: MockServer) -> None: run = CrawlerRun(DupeFilterSpider) await run.run(mockserver) @@ -400,14 +400,14 @@ class TestEngine(TestEngineBase): self._assert_dropped_requests(run) @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler_itemerror(self, mockserver: MockServer) -> None: run = CrawlerRun(ItemZeroDivisionErrorSpider) await run.run(mockserver) self._assert_items_error(run) @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler_change_close_reason_on_idle( self, mockserver: MockServer ) -> None: @@ -418,7 +418,7 @@ class TestEngine(TestEngineBase): "reason": "custom_reason", } == run.signals_caught[signals.spider_closed] - @deferred_f_from_coro_f + @coroutine_test async def test_close_downloader(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) await e.close_async() @@ -436,7 +436,7 @@ class TestEngine(TestEngineBase): get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None ) - @inlineCallbacks + @inline_callbacks_test def test_start_already_running_exception(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() @@ -449,7 +449,7 @@ class TestEngine(TestEngineBase): yield deferred_from_coro(e.stop_async()) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_start_already_running_exception_asyncio(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() @@ -460,7 +460,7 @@ class TestEngine(TestEngineBase): await asyncio.gather(e.start_async(), e.start_async()) await e.stop_async() - @inlineCallbacks + @inline_callbacks_test def test_start_request_processing_exception(self): class BadRequestFingerprinter: def fingerprint(self, request): @@ -527,7 +527,7 @@ class TestEngineDownloadAsync: async def _download(engine: ExecutionEngine, request: Request) -> Response: return await engine.download_async(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_async_success(self, engine): """Test basic successful async download of a request.""" request = Request("http://example.com") @@ -543,7 +543,7 @@ class TestEngineDownloadAsync: engine._slot.remove_request.assert_called_once_with(request) engine.downloader.fetch.assert_called_once_with(request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_async_redirect(self, engine): """Test async download with a redirect request.""" original_request = Request("http://example.com") @@ -569,7 +569,7 @@ class TestEngineDownloadAsync: [call(original_request), call(redirect_request)] ) - @deferred_f_from_coro_f + @coroutine_test async def test_download_async_no_spider(self, engine): """Test async download attempt when no spider is available.""" request = Request("http://example.com") @@ -577,7 +577,7 @@ class TestEngineDownloadAsync: with pytest.raises(RuntimeError, match="No open spider to crawl:"): await self._download(engine, request) - @deferred_f_from_coro_f + @coroutine_test async def test_download_async_failure(self, engine): """Test async download when the downloader raises an exception.""" request = Request("http://example.com") @@ -647,7 +647,7 @@ class TestEngineCloseSpider: crawler.spider = crawler._create_spider() return crawler - @deferred_f_from_coro_f + @coroutine_test async def test_no_slot(self, crawler: Crawler) -> None: engine = ExecutionEngine(crawler, lambda _: None) crawler.engine = engine @@ -660,14 +660,14 @@ class TestEngineCloseSpider: engine._slot = slot await engine.close_spider_async() - @deferred_f_from_coro_f + @coroutine_test async def test_no_spider(self, crawler: Crawler) -> None: engine = ExecutionEngine(crawler, lambda _: None) with pytest.raises(RuntimeError, match="Spider not opened"): await engine.close_spider_async() engine.downloader.close() # cleanup - @deferred_f_from_coro_f + @coroutine_test async def test_exception_slot( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -679,7 +679,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Slot close failure" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_downloader( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -690,7 +690,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Downloader close failure" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_scraper( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -701,7 +701,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Scraper close failure" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_scheduler( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -713,7 +713,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Scheduler close failure" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_signal( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -731,7 +731,7 @@ class TestEngineCloseSpider: reason="cancelled", ) - @deferred_f_from_coro_f + @coroutine_test async def test_exception_stats( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -742,7 +742,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Stats close failure" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_callback( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: @@ -752,7 +752,7 @@ class TestEngineCloseSpider: await engine.close_spider_async() assert "Error running spider_closed_callback" in caplog.text - @deferred_f_from_coro_f + @coroutine_test async def test_exception_async_callback( self, crawler: Crawler, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index f12409029..9bdbfcc56 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -12,7 +12,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.test_scheduler import MemoryScheduler -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.http import Response @@ -28,7 +28,7 @@ async def sleep(seconds: float = 0.001) -> None: class TestMain: @pytest.mark.requires_reactor # TODO - @deferred_f_from_coro_f + @coroutine_test async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on the scheduler (returning no requests while also returning True from @@ -87,7 +87,7 @@ class TestMain: expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - @deferred_f_from_coro_f + @coroutine_test async def test_close_during_start_iteration( self, caplog: pytest.LogCaptureFixture ) -> None: @@ -188,7 +188,7 @@ class TestRequestSendOrder: expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - @deferred_f_from_coro_f + @coroutine_test async def test_default(self): """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a @@ -228,7 +228,7 @@ class TestRequestSendOrder: parse_fn=parse, ) - @deferred_f_from_coro_f + @coroutine_test async def test_lifo_start(self): """Changing the queues of start requests to LIFO, matching the queues of non-start requests, does not cause all requests to be stored in the @@ -271,7 +271,7 @@ class TestRequestSendOrder: parse_fn=parse, ) - @deferred_f_from_coro_f + @coroutine_test async def test_shared_queues(self): """If SCHEDULER_START_*_QUEUE is falsy, start requests and other requests share the same queue, i.e. start requests are not priorized @@ -333,7 +333,7 @@ class TestRequestSendOrder: # spiders. @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_lazy(self): start_nums = [1, 2, 4] cb_nums = [3] diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index bd29e9ef6..970fffbc9 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -13,7 +13,7 @@ from tests.test_engine import ( MySpider, TestEngineBase, ) -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from tests.mockserver.http import MockServer @@ -27,7 +27,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 7dbaa41d2..e0008a53a 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -13,7 +13,7 @@ from tests.test_engine import ( MySpider, TestEngineBase, ) -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from tests.mockserver.http import MockServer @@ -27,7 +27,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index fcb027832..a956ab184 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -4,7 +4,7 @@ from twisted.cred import credentials from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test pytestmark = pytest.mark.requires_reactor @@ -23,7 +23,7 @@ class TestTelnetExtension: return console, portal - @inlineCallbacks + @inline_callbacks_test def test_bad_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword(b"username", b"password") @@ -32,7 +32,7 @@ class TestTelnetExtension: yield d console.stop_listening() - @inlineCallbacks + @inline_callbacks_test def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( @@ -42,7 +42,7 @@ class TestTelnetExtension: yield d console.stop_listening() - @inlineCallbacks + @inline_callbacks_test def test_custom_credentials(self): settings = { "TELNETCONSOLE_USERNAME": "user", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c65bd9df9..3a9c37042 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -55,7 +55,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer from tests.mockserver.http import MockServer from tests.spiders import ItemSpider -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -192,7 +192,7 @@ class TestFTPFeedStorage: finally: path.unlink() - @deferred_f_from_coro_f + @coroutine_test async def test_append(self): with MockFTPServer() as ftp_server: filename = "file" @@ -202,7 +202,7 @@ class TestFTPFeedStorage: await self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @deferred_f_from_coro_f + @coroutine_test async def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" @@ -211,7 +211,7 @@ class TestFTPFeedStorage: await self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @deferred_f_from_coro_f + @coroutine_test async def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -222,7 +222,7 @@ class TestFTPFeedStorage: await self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @deferred_f_from_coro_f + @coroutine_test async def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -314,7 +314,7 @@ class TestS3FeedStorage: assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @deferred_f_from_coro_f + @coroutine_test async def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -455,7 +455,7 @@ class TestS3FeedStorage: assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @deferred_f_from_coro_f + @coroutine_test async def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -475,7 +475,7 @@ class TestS3FeedStorage: ) assert acl is None - @deferred_f_from_coro_f + @coroutine_test async def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" @@ -540,7 +540,7 @@ class TestGCSFeedStorage: storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @deferred_f_from_coro_f + @coroutine_test async def test_store(self): try: from google.cloud.storage import Client # noqa: F401,PLC0415 @@ -1006,7 +1006,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["marshal"], load_func=marshal.load) assert result == expected - @inlineCallbacks + @inline_callbacks_test def test_stats_file_success(self): settings = { "FEEDS": { @@ -1020,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 - @inlineCallbacks + @inline_callbacks_test def test_stats_file_failed(self): settings = { "FEEDS": { @@ -1038,7 +1038,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 - @inlineCallbacks + @inline_callbacks_test def test_stats_multiple_file(self): settings = { "FEEDS": { @@ -1060,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items(self): # feed exporters use field names from Item items = [ @@ -1074,7 +1074,7 @@ class TestFeedExport(TestFeedExportBase): header = self.MyItem.fields.keys() await self.assertExported(items, header, rows) - @deferred_f_from_coro_f + @coroutine_test async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -1086,7 +1086,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_no_data(settings) assert data[fmt] is None - @deferred_f_from_coro_f + @coroutine_test async def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1106,7 +1106,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @deferred_f_from_coro_f + @coroutine_test async def test_start_finish_exporting_no_items(self): items = [] settings = { @@ -1124,7 +1124,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @deferred_f_from_coro_f + @coroutine_test async def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1145,7 +1145,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @deferred_f_from_coro_f + @coroutine_test async def test_start_finish_exporting_no_items_exception(self): items = [] settings = { @@ -1164,7 +1164,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @deferred_f_from_coro_f + @coroutine_test async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -1184,7 +1184,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_no_data(settings) assert expctd == data[fmt] - @deferred_f_from_coro_f + @coroutine_test async def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { @@ -1202,7 +1202,7 @@ class TestFeedExport(TestFeedExportBase): assert str(log).count("Storage.store is called") == 0 - @deferred_f_from_coro_f + @coroutine_test async def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1224,7 +1224,7 @@ class TestFeedExport(TestFeedExportBase): await self.assertExportedCsv(items, header, rows_csv) await self.assertExportedJsonLines(items, rows_jl) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] @@ -1234,7 +1234,7 @@ class TestFeedExport(TestFeedExportBase): await self.assertExportedCsv(items, header, rows) await self.assertExportedJsonLines(items, rows, settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1242,7 +1242,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} await self.assertExported(items, header, rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1250,7 +1250,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ",".join(header)} await self.assertExported(items, header, rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1258,7 +1258,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} await self.assertExported(items, header, rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1266,7 +1266,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} await self.assertExported(items, list(header.values()), rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { @@ -1277,7 +1277,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1285,7 +1285,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} await self.assertExported(items, list(header.values()), rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1331,7 +1331,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @deferred_f_from_coro_f + @coroutine_test async def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1390,7 +1390,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @deferred_f_from_coro_f + @coroutine_test async def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. @@ -1403,7 +1403,7 @@ class TestFeedExport(TestFeedExportBase): await self.assertExportedCsv(items, ["foo", "egg"], rows_csv) await self.assertExportedJsonLines(items, rows_jl) - @deferred_f_from_coro_f + @coroutine_test async def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, @@ -1414,7 +1414,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] await self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1440,7 +1440,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] await self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @deferred_f_from_coro_f + @coroutine_test async def test_export_encoding(self): items = [{"foo": "Test\xd6"}] @@ -1485,7 +1485,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_data(items, settings) assert data[fmt] == expected - @deferred_f_from_coro_f + @coroutine_test async def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1525,7 +1525,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @deferred_f_from_coro_f + @coroutine_test async def test_export_indentation(self): items = [ {"foo": ["bar"]}, @@ -1681,7 +1681,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @deferred_f_from_coro_f + @coroutine_test async def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, @@ -1694,7 +1694,7 @@ class TestFeedExport(TestFeedExportBase): assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @deferred_f_from_coro_f + @coroutine_test async def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, @@ -1703,7 +1703,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_no_data(settings) assert data["csv"] == b"" - @deferred_f_from_coro_f + @coroutine_test async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1723,7 +1723,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @deferred_f_from_coro_f + @coroutine_test async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1743,7 +1743,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @deferred_f_from_coro_f + @coroutine_test async def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1780,7 +1780,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @deferred_f_from_coro_f + @coroutine_test async def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1802,7 +1802,7 @@ class TestFeedExport(TestFeedExportBase): await self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @deferred_f_from_coro_f + @coroutine_test async def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1901,7 +1901,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @deferred_f_from_coro_f + @coroutine_test async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") @@ -1920,7 +1920,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid gzip data.") - @deferred_f_from_coro_f + @coroutine_test async def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( @@ -1957,7 +1957,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( @@ -1992,7 +1992,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( @@ -2027,7 +2027,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") @@ -2046,7 +2046,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @deferred_f_from_coro_f + @coroutine_test async def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( @@ -2079,7 +2079,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( @@ -2112,7 +2112,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( @@ -2145,7 +2145,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 @@ -2170,7 +2170,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = lzma.decompress(data[filename]) assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") @@ -2189,7 +2189,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid bz2 data.") - @deferred_f_from_coro_f + @coroutine_test async def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( @@ -2222,7 +2222,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_custom_plugin(self): filename = self._named_tempfile("csv_file") @@ -2238,7 +2238,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = await self.exported_data(self.items, settings) assert data[filename] == self.expected - @deferred_f_from_coro_f + @coroutine_test async def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2256,7 +2256,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = await self.exported_data(self.items, settings) assert data[filename] == expected - @deferred_f_from_coro_f + @coroutine_test async def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" @@ -2301,7 +2301,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = decompressor(data[filename]) assert result == expected - @deferred_f_from_coro_f + @coroutine_test async def test_exports_compatibility_with_postproc(self): filename_to_expected = { self._named_tempfile("csv"): b"foo\r\nbar\r\n", @@ -2511,7 +2511,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @deferred_f_from_coro_f + @coroutine_test async def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ @@ -2540,7 +2540,7 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @deferred_f_from_coro_f + @coroutine_test async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -2556,7 +2556,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert len(data[fmt]) == 0 - @deferred_f_from_coro_f + @coroutine_test async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -2580,7 +2580,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert data[fmt][0] == expctd - @deferred_f_from_coro_f + @coroutine_test async def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, @@ -2636,7 +2636,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt], strict=False): assert got_batch == expected_batch - @deferred_f_from_coro_f + @coroutine_test async def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { @@ -2660,7 +2660,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt], strict=False): assert got_batch == expected_batch - @deferred_f_from_coro_f + @coroutine_test async def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. @@ -2682,7 +2682,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = await self.exported_data(items, settings) assert len(items) == len(data["json"]) - @inlineCallbacks + @inline_callbacks_test def test_stats_batch_file_success(self): settings = { "FEEDS": { @@ -2700,7 +2700,7 @@ class TestBatchDeliveries(TestFeedExportBase): assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 - @inlineCallbacks + @inline_callbacks_test def test_s3_export(self): bucket = "mybucket" items = [ @@ -2823,7 +2823,7 @@ class TestFeedExporterSignals: feed_exporter.item_scraped(item, spider) await feed_exporter.close_spider(spider) - @deferred_f_from_coro_f + @coroutine_test async def test_feed_exporter_signals_sent(self) -> None: self.feed_exporter_closed_received = False self.feed_slot_closed_received = False @@ -2835,7 +2835,7 @@ class TestFeedExporterSignals: assert self.feed_slot_closed_received assert self.feed_exporter_closed_received - @deferred_f_from_coro_f + @coroutine_test async def test_feed_exporter_signals_sent_async(self) -> None: self.feed_exporter_closed_received = False self.feed_slot_closed_received = False diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 0602be0c8..66544a508 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -12,7 +12,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ItemSpider -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class CustomItem(Item): @@ -272,7 +272,7 @@ class TestShowOrSkipMessages: }, } - @inlineCallbacks + @inline_callbacks_test def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: @@ -281,7 +281,7 @@ class TestShowOrSkipMessages: assert "Crawled (200) None: crawler = self._create_crawler(pipeline_class) await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 1 - @deferred_f_from_coro_f + @coroutine_test async def test_pipeline_deferred(self, mockserver: MockServer) -> None: crawler = self._create_crawler(DeferredPipeline) with ( @@ -180,7 +180,7 @@ class TestPipeline: await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 1 - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None: crawler = self._create_crawler(DeprecatedSpiderArgPipeline) with ( @@ -214,7 +214,7 @@ class TestPipeline: ProcessItemExceptionAsyncPipeline, ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_process_item_exception( self, caplog: pytest.LogCaptureFixture, @@ -239,7 +239,7 @@ class TestPipeline: OpenSpiderExceptionAsyncPipeline, ], ) - @deferred_f_from_coro_f + @coroutine_test async def test_open_spider_exception( self, mockserver: MockServer, pipeline_class: type ) -> None: @@ -267,7 +267,7 @@ class TestCustomPipelineManager: itemproc.process_item({}, crawler.spider) @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_integration_recommended(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): async def process_item_async(self, item): @@ -294,7 +294,7 @@ class TestCustomPipelineManager: assert len(items) == 1 @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): def open_spider(self, spider): @@ -353,7 +353,7 @@ class TestCustomPipelineManager: assert len(items) == 1 @pytest.mark.requires_http_handler - @deferred_f_from_coro_f + @coroutine_test async def test_integration_no_async_not_subclass( self, mockserver: MockServer ) -> None: @@ -425,7 +425,7 @@ class TestMiddlewareManagerSpider: def crawler(self) -> Crawler: return get_crawler(Spider) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg_no_crawler_spider( self, crawler: Crawler ) -> None: @@ -480,7 +480,7 @@ class TestMiddlewareManagerSpider: ): await mwman.close_spider_async() - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None: """Crawler is provided and has a spider, works. The instance passed to a deprecated method is ignored, even if mismatched.""" @@ -497,7 +497,7 @@ class TestMiddlewareManagerSpider: ): await maybe_deferred_to_future(mwman.close_spider(DefaultSpider())) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg_without_crawler(self) -> None: """The first instance passed to a deprecated method is used. Mismatched ones raise an error.""" with pytest.warns( @@ -527,7 +527,7 @@ class TestMiddlewareManagerSpider: ): await maybe_deferred_to_future(mwman.close_spider(spider)) - @deferred_f_from_coro_f + @coroutine_test async def test_no_spider_arg_without_crawler(self) -> None: """If no crawler and no spider arg, raise an error.""" with pytest.warns( diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index daa635b54..c78c882cf 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -13,7 +13,7 @@ from scrapy.http import Request from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SimpleSpider, SingleRequestSpider -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class MitmProxy: @@ -84,14 +84,14 @@ class TestProxyConnect: self._proxy.stop() os.environ = self._oldenv - @inlineCallbacks + @inline_callbacks_test def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @inlineCallbacks + @inline_callbacks_test def test_https_tunnel_auth_error(self): os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"]) crawler = get_crawler(SimpleSpider) @@ -101,7 +101,7 @@ class TestProxyConnect: # he just sees a TunnelError. self._assert_got_tunnel_error(log) - @inlineCallbacks + @inline_callbacks_test def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 48234adb3..66dd48737 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -6,7 +6,7 @@ from scrapy.http.response import Response from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SingleRequestSpider -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test OVERRIDDEN_URL = "https://example.org" @@ -74,7 +74,7 @@ class TestCrawl: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_response_200(self): url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) @@ -82,7 +82,7 @@ class TestCrawl: response = crawler.spider.meta["responses"][0] assert response.request.url == url - @inlineCallbacks + @inline_callbacks_test def test_response_error(self): for status in ("404", "500"): url = self.mockserver.url(f"/status?n={status}") @@ -93,7 +93,7 @@ class TestCrawl: assert failure.request.url == url assert response.request.url == url - @inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_raise_exception(self): url = self.mockserver.url("/status?n=200") crawler = get_crawler( @@ -109,7 +109,7 @@ class TestCrawl: assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) - @inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_override_request_in_process_response(self): """ Downloader middleware which returns a response with an specific 'request' attribute. @@ -152,7 +152,7 @@ class TestCrawl: ), ) - @inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_override_in_process_exception(self): """ An exception is raised but caught by the next middleware, which @@ -175,7 +175,7 @@ class TestCrawl: assert response.body == b"Caught ZeroDivisionError" assert response.request.url == OVERRIDDEN_URL - @inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_do_not_override_in_process_exception(self): """ An exception is raised but caught by the next middleware, which @@ -198,7 +198,7 @@ class TestCrawl: assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url - @inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_alternative_callback(self): """ Downloader middleware which returns a response with a diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c91a670c1..c8629df00 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -5,7 +5,7 @@ from scrapy.http import Request from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class InjectArgumentsDownloaderMiddleware: @@ -160,7 +160,7 @@ class TestCallbackKeywordArguments: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_callback_kwargs(self): crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: diff --git a/tests/test_request_left.py b/tests/test_request_left.py index a5d5efffa..726e0573a 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -2,7 +2,7 @@ from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class SignalCatcherSpider(Spider): @@ -32,25 +32,25 @@ class TestCatching: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_success(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/status?n=200")) assert crawler.spider.caught_times == 1 - @inlineCallbacks + @inline_callbacks_test def test_timeout(self): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) assert crawler.spider.caught_times == 1 - @inlineCallbacks + @inline_callbacks_test def test_disconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/drop")) assert crawler.spider.caught_times == 1 - @inlineCallbacks + @inline_callbacks_test def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f363b1cd5..873be0b8f 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -19,7 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class MemoryScheduler(BaseScheduler): @@ -370,7 +370,7 @@ class TestIntegrationWithDownloaderAwareInMemory: ) @pytest.mark.requires_http_handler - @inlineCallbacks + @inline_callbacks_test def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: url = mockserver.url("/status?n=200", is_secure=False) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 277c00623..6234bef86 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -13,7 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] @@ -118,7 +118,7 @@ class TestSimpleScheduler(InterfaceCheckMixin): def setup_method(self): self.scheduler = SimpleScheduler() - @inlineCallbacks + @inline_callbacks_test def test_enqueue_dequeue(self): open_result = yield self.scheduler.open(Spider("foo")) assert open_result == "open" @@ -148,7 +148,7 @@ class TestSimpleScheduler(InterfaceCheckMixin): class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler - @inlineCallbacks + @inline_callbacks_test def test_crawl(self): with MockServer() as mockserver: settings = { diff --git a/tests/test_signals.py b/tests/test_signals.py index 5769f6588..faf24a129 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -3,7 +3,7 @@ import pytest from scrapy import Request, Spider, signals from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver.http import MockServer -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test class ItemSpider(Spider): @@ -20,7 +20,7 @@ class ItemSpider(Spider): class TestMain: - @deferred_f_from_coro_f + @coroutine_test async def test_scheduler_empty(self): crawler = get_crawler() calls = [] @@ -52,7 +52,7 @@ class TestMockServer: @pytest.mark.requires_http_handler @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) diff --git a/tests/test_spider.py b/tests/test_spider.py index dd1cc15b0..68d4f85db 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -30,7 +30,7 @@ from scrapy.spiders import ( from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir -from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks +from tests.utils.decorators import coroutine_test, inline_callbacks_test class TestSpider: @@ -95,7 +95,7 @@ class TestSpider: assert settings.get("TEST2") == "spider" assert settings.get("TEST3") == "project" - @inlineCallbacks + @inline_callbacks_test def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} project_settings = { @@ -142,7 +142,7 @@ class TestSpider: class TestInitSpider(TestSpider): spider_class = InitSpider - @deferred_f_from_coro_f + @coroutine_test async def test_start_urls(self): responses = [] @@ -458,7 +458,7 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links - @inlineCallbacks + @inline_callbacks_test def test_start_url(self): class TestSpider(self.spider_class): name = "test" @@ -827,7 +827,7 @@ Sitemap: /sitemap-relative-url.xml ), ) - @deferred_f_from_coro_f + @coroutine_test async def test_sitemap_urls(self): class TestSpider(self.spider_class): name = "test" diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index b84b0d8ac..8ca588978 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -13,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from .utils import twisted_sleep -from .utils.decorators import deferred_f_from_coro_f +from .utils.decorators import coroutine_test SLEEP_SECONDS = 0.1 @@ -38,7 +38,7 @@ class TestMain: assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items - @deferred_f_from_coro_f + @coroutine_test async def test_start_urls(self): class TestSpider(Spider): name = "test" @@ -51,7 +51,7 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_start(self): class TestSpider(Spider): name = "test" @@ -63,7 +63,7 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_start_subclass(self): class BaseSpider(Spider): async def start(self): @@ -76,7 +76,7 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated(self): class TestSpider(Spider): name = "test" @@ -87,7 +87,7 @@ class TestMain: with pytest.warns(ScrapyDeprecationWarning): await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_subclass(self): class BaseSpider(Spider): def start_requests(self): @@ -100,7 +100,7 @@ class TestMain: with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_universal(self): class TestSpider(Spider): name = "test" @@ -115,7 +115,7 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_subclass(self): class BaseSpider(Spider): async def start(self): @@ -131,7 +131,7 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f + @coroutine_test async def test_start_deprecated_super(self): class TestSpider(Spider): name = "test" @@ -154,7 +154,7 @@ class TestMain: await self._test_spider(TestSpider, expected_items) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_asyncio_delayed(self): async def start(spider): await sleep(SLEEP_SECONDS) @@ -163,7 +163,7 @@ class TestMain: await self._test_start(start, [ITEM_A]) @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() - @deferred_f_from_coro_f + @coroutine_test async def test_twisted_delayed(self): async def start(spider): await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) @@ -173,7 +173,7 @@ class TestMain: # Exceptions - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_non_generator_exception(self): class TestSpider(Spider): name = "test" diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 17d897526..9651e3f8b 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -18,7 +18,7 @@ from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.python.failure import Failure @@ -53,7 +53,7 @@ class TestSpiderMiddleware: class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response): @@ -67,7 +67,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result): @@ -81,7 +81,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - @deferred_f_from_coro_f + @coroutine_test async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception): @@ -100,7 +100,7 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - @deferred_f_from_coro_f + @coroutine_test async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception): @@ -237,47 +237,47 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - @deferred_f_from_coro_f + @coroutine_test async def test_simple(self): """Simple mw""" await self._test_simple_base(self.MW_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_asyncgen(self): """Asyncgen mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_asyncgen_simple(self): """Asyncgen mw -> simple mw; upgrade then downgrade""" await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - @deferred_f_from_coro_f + @coroutine_test async def test_universal(self): """Universal mw""" await self._test_simple_base(self.MW_UNIVERSAL) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_simple(self): """Universal mw -> simple mw""" await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_universal(self): """Simple mw -> universal mw""" await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_asyncgen(self): """Universal mw -> asyncgen mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) - @deferred_f_from_coro_f + @coroutine_test async def test_asyncgen_universal(self): """Asyncgen mw -> universal mw; upgrade""" await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) @@ -290,27 +290,27 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): for item in super()._callback(): yield item - @deferred_f_from_coro_f + @coroutine_test async def test_simple(self): """Simple mw; downgrade""" await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; downgrade then upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) - @deferred_f_from_coro_f + @coroutine_test async def test_universal(self): """Universal mw""" await self._test_asyncgen_base(self.MW_UNIVERSAL) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_simple(self): """Universal mw -> simple mw; downgrade""" await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_universal(self): """Simple mw -> universal mw; downgrade""" await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) @@ -327,7 +327,7 @@ class ProcessSpiderOutputCoroutineMiddleware: class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): - @deferred_f_from_coro_f + @coroutine_test async def test_non_iterable(self): with pytest.raises( _InvalidOutput, @@ -335,7 +335,7 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ): await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) - @deferred_f_from_coro_f + @coroutine_test async def test_coroutine(self): with pytest.raises( _InvalidOutput, @@ -375,7 +375,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) return await self.mwman.process_start() - @deferred_f_from_coro_f + @coroutine_test async def test_simple(self): """Simple mw""" start = await self._get_processed_start(self.MW_SIMPLE) @@ -489,33 +489,33 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): self._scrape_func, self.response, self.request ) - @deferred_f_from_coro_f + @coroutine_test async def test_just_builtin(self): await self._test_simple_base() - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_simple(self): await self._test_simple_base(self.MW_SIMPLE, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_async(self): """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_universal(self): await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_builtin(self): await self._test_simple_base(self.MW_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_async_builtin(self): """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_builtin(self): await self._test_simple_base(self.MW_UNIVERSAL) @@ -525,33 +525,33 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): for item in super()._callback(): yield item - @deferred_f_from_coro_f + @coroutine_test async def test_just_builtin(self): await self._test_asyncgen_base() - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_simple(self): """Downgrade""" await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_async(self): await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_builtin_universal(self): await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - @deferred_f_from_coro_f + @coroutine_test async def test_simple_builtin(self): """Downgrade""" await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - @deferred_f_from_coro_f + @coroutine_test async def test_async_builtin(self): await self._test_asyncgen_base(self.MW_ASYNCGEN) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_builtin(self): await self._test_asyncgen_base(self.MW_UNIVERSAL) @@ -574,39 +574,39 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ): await self._get_middleware_result(*mw_classes) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_simple(self): """Simple exc mw""" await self._test_simple_base(self.MW_EXC_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_async(self): """Async exc mw""" await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_simple_simple(self): """Simple exc mw -> simple output mw""" await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_async_async(self): """Async exc mw -> async output mw""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_simple_async(self): """Simple exc mw -> async output mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - @deferred_f_from_coro_f + @coroutine_test async def test_exc_async_simple(self): """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) class TestDeprecatedSpiderArg(TestSpiderMiddleware): - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mw_spider_arg(self): class DeprecatedSpiderArgMiddleware: def process_spider_input(self, response, spider): @@ -635,7 +635,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware): self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) await self._scrape_response() - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mwman_spider_arg(self): with pytest.warns( ScrapyDeprecationWarning, @@ -644,7 +644,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware): ): await self.mwman.process_start(DefaultSpider()) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mwman_spider_arg_no_crawler(self): with pytest.warns( ScrapyDeprecationWarning, diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index d4d0a4fb1..8dc6ff300 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class _HttpErrorSpider(MockServerSpider): @@ -202,7 +202,7 @@ class TestHttpErrorMiddlewareIntegrational: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks + @inline_callbacks_test def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -216,7 +216,7 @@ class TestHttpErrorMiddlewareIntegrational: assert get_value("httperror/response_ignored_status_count/402") == 1 assert get_value("httperror/response_ignored_status_count/500") == 1 - @inlineCallbacks + @inline_callbacks_test def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: @@ -230,7 +230,7 @@ class TestHttpErrorMiddlewareIntegrational: assert "Ignoring response <200" not in str(log) assert "Ignoring response <402" not in str(log) - @inlineCallbacks + @inline_callbacks_test def test_logging_level(self): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 07df2ecc4..80f5828c9 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -4,7 +4,7 @@ from testfixtures import LogCapture from scrapy import Request, Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test class _BaseSpiderMiddleware: @@ -338,7 +338,7 @@ class TestSpiderMiddleware: await crawler.crawl_async(mockserver=self.mockserver) return log - @deferred_f_from_coro_f + @coroutine_test async def test_recovery(self): """ (0) Recover from an exception in a spider callback. The final item count should be 3 @@ -351,7 +351,7 @@ class TestSpiderMiddleware: assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @deferred_f_from_coro_f + @coroutine_test async def test_recovery_asyncgen(self): """ Same as test_recovery but with an async callback. @@ -361,7 +361,7 @@ class TestSpiderMiddleware: assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @deferred_f_from_coro_f + @coroutine_test async def test_process_spider_input_without_errback(self): """ (1.1) An exception from the process_spider_input chain should be caught by the @@ -371,7 +371,7 @@ class TestSpiderMiddleware: assert "Middleware: will raise IndexError" in str(log1) assert "Middleware: IndexError exception caught" in str(log1) - @deferred_f_from_coro_f + @coroutine_test async def test_process_spider_input_with_errback(self): """ (1.2) An exception from the process_spider_input chain should not be caught by the @@ -385,7 +385,7 @@ class TestSpiderMiddleware: assert "{'from': 'callback'}" not in str(log1) assert "'item_scraped_count': 1" in str(log1) - @deferred_f_from_coro_f + @coroutine_test async def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should @@ -396,7 +396,7 @@ class TestSpiderMiddleware: assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @deferred_f_from_coro_f + @coroutine_test async def test_async_generator_callback(self): """ Same as test_generator_callback but with an async callback. @@ -405,7 +405,7 @@ class TestSpiderMiddleware: assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @deferred_f_from_coro_f + @coroutine_test async def test_generator_callback_right_after_callback(self): """ (2.1) Special case of (2): Exceptions should be caught @@ -415,7 +415,7 @@ class TestSpiderMiddleware: assert "Middleware: ImportError exception caught" in str(log21) assert "'item_scraped_count': 2" in str(log21) - @deferred_f_from_coro_f + @coroutine_test async def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should @@ -425,7 +425,7 @@ class TestSpiderMiddleware: assert "Middleware: ZeroDivisionError exception caught" in str(log3) assert "item_scraped_count" not in str(log3) - @deferred_f_from_coro_f + @coroutine_test async def test_not_a_generator_callback_right_after_callback(self): """ (3.1) Special case of (3): Exceptions should be caught @@ -437,7 +437,7 @@ class TestSpiderMiddleware: assert "Middleware: ZeroDivisionError exception caught" in str(log31) assert "item_scraped_count" not in str(log31) - @deferred_f_from_coro_f + @coroutine_test async def test_generator_output_chain(self): """ (4) An exception from a middleware's process_spider_output method should be sent @@ -484,7 +484,7 @@ class TestSpiderMiddleware: assert str(item_recovered) in str(log4) assert "parse-second-item" not in str(log4) - @deferred_f_from_coro_f + @coroutine_test async def test_not_a_generator_output_chain(self): """ (5) An exception from a middleware's process_spider_output method should be sent diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index e7c9640b8..bccfc4432 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler from tests.test_spider_start import SLEEP_SECONDS from .utils import twisted_sleep -from .utils.decorators import deferred_f_from_coro_f +from .utils.decorators import coroutine_test ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} @@ -130,45 +130,45 @@ class TestMain: expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] await self._test([smw1, smw2], spider_cls, expected_items) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_modern_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_universal_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_deprecated_spider(self): with pytest.warns( ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" ): await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_modern_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_universal_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_deprecated_spider(self): with pytest.warns( ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" ): await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mw_modern_spider(self): with ( pytest.warns( @@ -180,7 +180,7 @@ class TestMain: ): await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mw_modern_spider_subclass(self): with ( pytest.warns( @@ -195,7 +195,7 @@ class TestMain: DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass ) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mw_universal_spider(self): with pytest.warns( ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" @@ -206,7 +206,7 @@ class TestMain: [ITEM_A, ITEM_D, ITEM_C], ) - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_mw_deprecated_spider(self): with ( pytest.warns( @@ -218,7 +218,7 @@ class TestMain: ): await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_universal_mw_modern_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") @@ -228,7 +228,7 @@ class TestMain: ModernWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_deprecated_mw_modern_spider(self): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): await self._test_douple_wrap( @@ -237,7 +237,7 @@ class TestMain: ModernWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_deprecated_mw_modern_spider(self): with ( pytest.warns( @@ -253,7 +253,7 @@ class TestMain: ModernWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_universal_mw_universal_spider(self): with warnings.catch_warnings(): warnings.simplefilter("error") @@ -263,7 +263,7 @@ class TestMain: UniversalWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_deprecated_mw_universal_spider(self): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): await self._test_douple_wrap( @@ -272,7 +272,7 @@ class TestMain: UniversalWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_deprecated_mw_universal_spider(self): with pytest.warns( ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" @@ -284,7 +284,7 @@ class TestMain: [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], ) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_universal_mw_deprecated_spider(self): with pytest.warns( ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" @@ -295,7 +295,7 @@ class TestMain: DeprecatedWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_modern_mw_deprecated_mw_deprecated_spider(self): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): await self._test_douple_wrap( @@ -304,7 +304,7 @@ class TestMain: DeprecatedWrapSpider, ) - @deferred_f_from_coro_f + @coroutine_test async def test_universal_mw_deprecated_mw_deprecated_spider(self): with ( pytest.warns( @@ -330,24 +330,24 @@ class TestMain: await self._test(spider_middlewares, TestSpider, [ITEM_A]) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_asyncio_sleep_single(self): await self._test_sleep([AsyncioSleepSpiderMiddleware]) @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_asyncio_sleep_multiple(self): await self._test_sleep( [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] ) @pytest.mark.requires_reactor - @deferred_f_from_coro_f + @coroutine_test async def test_twisted_sleep_single(self): await self._test_sleep([TwistedSleepSpiderMiddleware]) @pytest.mark.requires_reactor - @deferred_f_from_coro_f + @coroutine_test async def test_twisted_sleep_multiple(self): await self._test_sleep( [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 1f2209f60..76976d962 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -3,11 +3,11 @@ from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test class TestMiddleware: - @deferred_f_from_coro_f + @coroutine_test async def test_async(self): crawler = get_crawler(Spider) mw = build_from_crawler(StartSpiderMiddleware, crawler) @@ -24,7 +24,7 @@ class TestMiddleware: ] assert result == [True, True, False, "foo"] - @deferred_f_from_coro_f + @coroutine_test async def test_sync(self): crawler = get_crawler(Spider) mw = build_from_crawler(StartSpiderMiddleware, crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 187d240a8..2869d302e 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -12,7 +12,7 @@ from scrapy.spiders import Spider from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -121,7 +121,7 @@ class TestStatsCollector: ): assert stats.get_stats(spider) == {"test": "value"} - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg_custom_collector(self) -> None: class CustomStatsCollector: def __init__(self, crawler): @@ -153,7 +153,7 @@ class TestStatsCollector: ): await crawler.crawl_async(url="data:,") - @deferred_f_from_coro_f + @coroutine_test async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None: class CustomStatsCollector(StatsCollector): def open_spider(self, spider): # pylint: disable=signature-differs diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index dba1ac5da..fc4e1c487 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,15 +1,15 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test class TestAsyncgenUtils: - @deferred_f_from_coro_f + @coroutine_test async def test_as_async_generator(self): ag = as_async_generator(range(42)) results = [i async for i in ag] assert results == list(range(42)) - @deferred_f_from_coro_f + @coroutine_test async def test_collect_asyncgen(self): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 6706c1c80..5489fd948 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -14,7 +14,7 @@ from scrapy.utils.asyncio import ( _parallel_asyncio, is_asyncio_available, ) -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -67,7 +67,7 @@ class TestParallelAsyncio: await asyncio.sleep(random.random() / 20) yield i - @deferred_f_from_coro_f + @coroutine_test async def test_simple(self): for length in [20, 50, 100]: parallel_count = [0] @@ -85,7 +85,7 @@ class TestParallelAsyncio: assert list(range(length)) == sorted(results) assert max_parallel_count[0] <= self.CONCURRENT_ITEMS - @deferred_f_from_coro_f + @coroutine_test async def test_delays(self): for length in [20, 50, 100]: parallel_count = [0] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index ad4cc466d..296bd6224 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -6,8 +6,7 @@ from asyncio import Future from typing import TYPE_CHECKING, Any import pytest -from twisted.internet.defer import Deferred, succeed -from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig +from twisted.internet.defer import Deferred, inlineCallbacks, succeed from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( @@ -20,7 +19,7 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, ) -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator @@ -29,7 +28,7 @@ if TYPE_CHECKING: @pytest.mark.requires_reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestMustbeDeferred: - @inlineCallbacks + @inline_callbacks_test def test_success_function(self) -> Generator[Deferred[Any], Any, None]: steps: list[int] = [] @@ -45,7 +44,7 @@ class TestMustbeDeferred: steps.append(2) # add another value, that should be caught by assertEqual yield dfd - @inlineCallbacks + @inline_callbacks_test def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]: steps: list[int] = [] @@ -224,7 +223,7 @@ class TestParallelAsync: await maybe_deferred_to_future(dfd) yield i - @inlineCallbacks + @inline_callbacks_test def test_simple(self): for length in [20, 50, 100]: parallel_count = [0] @@ -244,7 +243,7 @@ class TestParallelAsync: assert parallel_count[0] == 0 assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] - @inlineCallbacks + @inline_callbacks_test def test_delays(self): for length in [20, 50, 100]: parallel_count = [0] @@ -276,7 +275,7 @@ class TestDeferredFromCoro: result = deferred_from_coro(42) assert result == 42 - @inlineCallbacks + @inline_callbacks_test def test_coroutine(self): async def coroutine() -> int: return 42 @@ -287,7 +286,7 @@ class TestDeferredFromCoro: assert coro_result == 42 @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_coroutine_asyncio(self): async def coroutine() -> int: await asyncio.sleep(0.01) @@ -299,7 +298,7 @@ class TestDeferredFromCoro: assert coro_result == 42 @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_future(self): future = Future() result = deferred_from_coro(future) @@ -310,7 +309,7 @@ class TestDeferredFromCoro: class TestDeferredFFromCoroF: - @inlineCallbacks_orig + @inlineCallbacks def _assert_result( self, c_f: Callable[[], Awaitable[int]] ) -> Generator[Deferred[Any], Any, None]: @@ -320,7 +319,7 @@ class TestDeferredFFromCoroF: result = yield d assert result == 42 - @inlineCallbacks + @inline_callbacks_test def test_coroutine(self): async def c_f() -> int: return 42 @@ -328,7 +327,7 @@ class TestDeferredFFromCoroF: yield self._assert_result(c_f) @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_coroutine_asyncio(self): async def c_f() -> int: await asyncio.sleep(0.01) @@ -337,7 +336,7 @@ class TestDeferredFFromCoroF: yield self._assert_result(c_f) @pytest.mark.only_asyncio - @inlineCallbacks + @inline_callbacks_test def test_future(self): def c_f() -> Future[int]: f: Future[int] = Future() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 4a9e375fb..8dfac5f02 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -20,7 +20,7 @@ from scrapy.utils.python import ( to_unicode, without_none_values, ) -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -64,7 +64,7 @@ class TestMutableAsyncChain: for i in range(5, 7): yield i - @deferred_f_from_coro_f + @coroutine_test async def test_mutableasyncchain(self): m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) m.extend(self.g2()) @@ -74,7 +74,7 @@ class TestMutableAsyncChain: results = await collect_asyncgen(m) assert results == list(range(1, 10)) - @deferred_f_from_coro_f + @coroutine_test async def test_mutableasyncchain_exc(self): m = MutableAsyncChain(self.g1()) m.extend(self.g4()) diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index a045c05a4..3255d5940 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -9,7 +9,7 @@ from scrapy.utils.reactor import ( is_asyncio_reactor_installed, set_asyncio_event_loop, ) -from tests.utils.decorators import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test class TestAsyncio: @@ -31,7 +31,7 @@ class TestAsyncio: @pytest.mark.requires_reactor @pytest.mark.only_asyncio - @deferred_f_from_coro_f + @coroutine_test async def test_set_asyncio_event_loop(self): install_reactor(_asyncio_reactor_path) assert set_asyncio_event_loop(None) is asyncio.get_running_loop() diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index e47e9013a..9ac0fd0c0 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -14,14 +14,14 @@ from scrapy.utils.signal import ( send_catch_log_deferred, ) from scrapy.utils.test import get_from_asyncio_queue -from tests.utils.decorators import inlineCallbacks +from tests.utils.decorators import inline_callbacks_test class TestSendCatchLog: # whether the function being tested returns exceptions or failures returns_exceptions: bool = False - @inlineCallbacks + @inline_callbacks_test def test_send_catch_log(self): test_signal = object() handlers_called = set() diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py index 633d8d97d..dd9043ab1 100644 --- a/tests/utils/decorators.py +++ b/tests/utils/decorators.py @@ -4,8 +4,7 @@ from functools import wraps from typing import TYPE_CHECKING, Any, ParamSpec import pytest -from twisted.internet.defer import Deferred -from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.reactor import is_reactor_installed @@ -17,7 +16,7 @@ if TYPE_CHECKING: _P = ParamSpec("_P") -def inlineCallbacks( +def inline_callbacks_test( f: Callable[_P, Generator[Deferred[Any], Any, None]], ) -> Callable[_P, Awaitable[None]]: """Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style. @@ -34,12 +33,12 @@ def inlineCallbacks( @pytest.mark.asyncio @wraps(f) async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None: - await deferred_to_future(inlineCallbacks_orig(f)(*args, **kwargs)) + await deferred_to_future(inlineCallbacks(f)(*args, **kwargs)) return wrapper_coro @wraps(f) - @inlineCallbacks_orig + @inlineCallbacks def wrapper_dfd( *args: _P.args, **kwargs: _P.kwargs ) -> Generator[Deferred[Any], Any, None]: @@ -48,7 +47,7 @@ def inlineCallbacks( return wrapper_dfd -def deferred_f_from_coro_f( +def coroutine_test( coro_f: Callable[_P, Awaitable[None]], ) -> Callable[_P, Awaitable[None]]: """Mark a test function that returns a coroutine.