diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.py b/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.py new file mode 100644 index 000000000..d3eb2bec1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(SleepingSpider) +process.start(stop_after_crawl=False) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 34f5aa18a..d0ead784b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -943,6 +943,25 @@ async def test_async_crawler_process_force_stop_reactor_enabled_via_public_crawl assert called +@pytest.mark.only_asyncio +@coroutine_test +async def test_async_crawler_process_force_stop_reactorless_without_main_task( + reactor_pytest: str, +) -> None: + if reactor_pytest != "none": + pytest.skip("This test is only for --reactor=none") + + crawler_process = AsyncCrawlerProcess( + {"TWISTED_REACTOR_ENABLED": False}, + install_root_handler=False, + ) + assert crawler_process._reactorless_loop is not None + assert crawler_process._reactorless_main_task is None + crawler = crawler_process.create_crawler(DefaultSpider) + + await crawler.stop_async(mode="force") + + @pytest.mark.only_asyncio @coroutine_test async def test_async_crawler_process_force_stop_reactorless_without_loop( @@ -998,3 +1017,48 @@ async def test_async_crawler_process_force_stop_reactorless_with_task( assert loop.callback is not None loop.callback() assert task.called is True + + +def test_async_crawler_process_schedule_reactorless_shutdown_without_loop() -> None: + crawler_process = object.__new__(AsyncCrawlerProcess) + crawler_process._reactorless_loop = None + + crawler_process._schedule_reactorless_shutdown(mode="graceful") + + +def test_async_crawler_process_schedule_reactorless_shutdown_runtime_error() -> None: + crawler_process = object.__new__(AsyncCrawlerProcess) + + class DummyLoop: + scheduled = False + create_task_called = False + + def call_soon_threadsafe(self, callback) -> None: + self.scheduled = True + callback() + + def create_task(self, coro) -> None: + self.create_task_called = True + raise RuntimeError("event loop is closing") + + class DummyCoro: + closed = False + + def close(self) -> None: + self.closed = True + + loop = DummyLoop() + coro = DummyCoro() + + def shutdown_reactorless(*, mode: str) -> DummyCoro: + assert mode == "graceful" + return coro + + crawler_process._reactorless_loop = cast("asyncio.AbstractEventLoop", loop) + crawler_process._shutdown_reactorless = shutdown_reactorless + + crawler_process._schedule_reactorless_shutdown(mode="graceful") + + assert loop.scheduled + assert loop.create_task_called + assert coro.closed diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index ec40d0488..4893537f8 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -420,6 +420,9 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def test_shutdown_graceful(self) -> None: self._test_shutdown_graceful("reactorless_sleeping.py") + def test_shutdown_graceful_stop_after_crawl_false(self) -> None: + self._test_shutdown_graceful("reactorless_sleeping_no_stop_after_crawl.py") + @coroutine_test async def test_shutdown_forced(self) -> None: await self._test_shutdown_forced("reactorless_sleeping.py") diff --git a/tests/test_engine.py b/tests/test_engine.py index 54366fba7..38047a35c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -484,6 +484,20 @@ class TestEngine(TestEngineBase): engine._closewait.callback(None) await maybe_deferred_to_future(stop_dfd) + @coroutine_test + async def test_stop_async_reentrant_graceful_without_spider_or_closewait( + self, + ) -> None: + engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None) + engine._stopping = True + + with patch.object( + engine, "close_spider_async", new_callable=AsyncMock + ) as close: + await engine.stop_async(mode="graceful") + + close.assert_not_called() + @coroutine_test async def test_handle_downloader_output_ignores_fast_cancelled_failures( self,