From 0bb4ab14825268cd2c9631706cf9e8e2e2f17ecb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Jul 2026 00:33:27 +0500 Subject: [PATCH] Split out test_engine_close_spider.py. --- tests/test_engine.py | 134 +-------------------------- tests/test_engine_close_spider.py | 146 ++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 133 deletions(-) create mode 100644 tests/test_engine_close_spider.py diff --git a/tests/test_engine.py b/tests/test_engine.py index 3a1bb582a..3ab8f6d1b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -6,7 +6,7 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from unittest.mock import Mock from urllib.parse import urlparse @@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Response from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider -from scrapy.statscollectors import MemoryStatsCollector from scrapy.utils.defer import ( _schedule_coro, deferred_from_coro, @@ -42,8 +41,6 @@ if TYPE_CHECKING: from twisted.python.failure import Failure - from scrapy.core.scheduler import Scheduler - from scrapy.crawler import Crawler from tests.mockserver.http import MockServer @@ -548,132 +545,3 @@ async def test_request_scheduled_signal(): f"{scheduler.enqueued!r} != [{keep_request!r}]" ) crawler.signals.disconnect(signal_handler, signals.request_scheduled) - - -class TestEngineCloseSpider: - """Tests for exception handling coverage during close_spider_async().""" - - @pytest.fixture - def crawler(self) -> Crawler: - crawler = get_crawler(DefaultSpider) - crawler.spider = crawler._create_spider() - return crawler - - @coroutine_test - async def test_no_slot(self, crawler: Crawler) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - slot = engine._slot - engine._slot = None - with pytest.raises(RuntimeError, match="Engine slot not assigned"): - await engine.close_spider_async() - # close it correctly - engine._slot = slot - await engine.close_spider_async() - - @coroutine_test - async def test_no_spider(self, crawler: Crawler) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - with pytest.raises(RuntimeError, match="Spider not opened"): - await engine.close_spider_async() - engine.downloader.close() # cleanup - - @coroutine_test - async def test_exception_slot( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert engine._slot - del engine._slot.heartbeat - await engine.close_spider_async() - assert "Slot close failure" in caplog.text - - @coroutine_test - async def test_exception_downloader( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - del engine.downloader.slots - await engine.close_spider_async() - assert "Downloader close failure" in caplog.text - - @coroutine_test - async def test_exception_scraper( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - engine.scraper.slot = None - await engine.close_spider_async() - assert "Scraper close failure" in caplog.text - - @coroutine_test - async def test_exception_scheduler( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert engine._slot - del cast("Scheduler", engine._slot.scheduler).dqs - await engine.close_spider_async() - assert "Scheduler close failure" in caplog.text - - @coroutine_test - async def test_exception_signal( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - signal_manager = engine.signals - del engine.signals - await engine.close_spider_async() - assert "Error while sending spider_close signal" in caplog.text - # send the spider_closed signal to close various components - await signal_manager.send_catch_log_async( - signal=signals.spider_closed, - spider=engine.spider, - reason="cancelled", - ) - - @coroutine_test - async def test_exception_stats( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert isinstance(crawler.stats, MemoryStatsCollector) - del crawler.stats.spider_stats - await engine.close_spider_async() - assert "Stats close failure" in caplog.text - - @coroutine_test - async def test_exception_callback( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError())) - crawler.engine = engine - await engine.open_spider_async() - await engine.close_spider_async() - assert "Error running spider_closed_callback" in caplog.text - - @coroutine_test - async def test_exception_async_callback( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - async def cb(_): - raise ValueError - - engine = ExecutionEngine(crawler, cb) - crawler.engine = engine - await engine.open_spider_async() - await engine.close_spider_async() - assert "Error running spider_closed_callback" in caplog.text diff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.py new file mode 100644 index 000000000..27feeb682 --- /dev/null +++ b/tests/test_engine_close_spider.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import pytest +from twisted.internet import defer + +from scrapy import signals +from scrapy.core.engine import ExecutionEngine +from scrapy.statscollectors import MemoryStatsCollector +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.core.scheduler import Scheduler + from scrapy.crawler import Crawler + + +class TestEngineCloseSpider: + """Tests for exception handling coverage during close_spider_async().""" + + @pytest.fixture + def crawler(self) -> Crawler: + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + return crawler + + @coroutine_test + async def test_no_slot(self, crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + slot = engine._slot + engine._slot = None + with pytest.raises(RuntimeError, match="Engine slot not assigned"): + await engine.close_spider_async() + # close it correctly + engine._slot = slot + await engine.close_spider_async() + + @coroutine_test + async def test_no_spider(self, crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + with pytest.raises(RuntimeError, match="Spider not opened"): + await engine.close_spider_async() + engine.downloader.close() # cleanup + + @coroutine_test + async def test_exception_slot( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del engine._slot.heartbeat + await engine.close_spider_async() + assert "Slot close failure" in caplog.text + + @coroutine_test + async def test_exception_downloader( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + del engine.downloader.slots + await engine.close_spider_async() + assert "Downloader close failure" in caplog.text + + @coroutine_test + async def test_exception_scraper( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + engine.scraper.slot = None + await engine.close_spider_async() + assert "Scraper close failure" in caplog.text + + @coroutine_test + async def test_exception_scheduler( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del cast("Scheduler", engine._slot.scheduler).dqs + await engine.close_spider_async() + assert "Scheduler close failure" in caplog.text + + @coroutine_test + async def test_exception_signal( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + signal_manager = engine.signals + del engine.signals + await engine.close_spider_async() + assert "Error while sending spider_close signal" in caplog.text + # send the spider_closed signal to close various components + await signal_manager.send_catch_log_async( + signal=signals.spider_closed, + spider=engine.spider, + reason="cancelled", + ) + + @coroutine_test + async def test_exception_stats( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert isinstance(crawler.stats, MemoryStatsCollector) + del crawler.stats.spider_stats + await engine.close_spider_async() + assert "Stats close failure" in caplog.text + + @coroutine_test + async def test_exception_callback( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError())) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text + + @coroutine_test + async def test_exception_async_callback( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + async def cb(_): + raise ValueError + + engine = ExecutionEngine(crawler, cb) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text