mirror of https://github.com/scrapy/scrapy.git
Unpack TestEngineCloseSpider.
This commit is contained in:
Andrey Rakhmatullin
parent
0bb4ab1482
commit
46f56b3f09
|
|
@ -17,130 +17,137 @@ if TYPE_CHECKING:
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
|
|
||||||
|
|
||||||
class TestEngineCloseSpider:
|
@pytest.fixture
|
||||||
"""Tests for exception handling coverage during close_spider_async()."""
|
def crawler() -> Crawler:
|
||||||
|
crawler = get_crawler(DefaultSpider)
|
||||||
|
crawler.spider = crawler._create_spider()
|
||||||
|
return crawler
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def crawler(self) -> Crawler:
|
|
||||||
crawler = get_crawler(DefaultSpider)
|
|
||||||
crawler.spider = crawler._create_spider()
|
|
||||||
return crawler
|
|
||||||
|
|
||||||
@coroutine_test
|
@coroutine_test
|
||||||
async def test_no_slot(self, crawler: Crawler) -> None:
|
async def test_no_slot(crawler: Crawler) -> None:
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
crawler.engine = engine
|
crawler.engine = engine
|
||||||
await engine.open_spider_async()
|
await engine.open_spider_async()
|
||||||
slot = engine._slot
|
slot = engine._slot
|
||||||
engine._slot = None
|
engine._slot = None
|
||||||
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
|
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
|
||||||
await engine.close_spider_async()
|
|
||||||
# close it correctly
|
|
||||||
engine._slot = slot
|
|
||||||
await engine.close_spider_async()
|
await engine.close_spider_async()
|
||||||
|
# close it correctly
|
||||||
|
engine._slot = slot
|
||||||
|
await engine.close_spider_async()
|
||||||
|
|
||||||
@coroutine_test
|
|
||||||
async def test_no_spider(self, crawler: Crawler) -> None:
|
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
|
||||||
with pytest.raises(RuntimeError, match="Spider not opened"):
|
|
||||||
await engine.close_spider_async()
|
|
||||||
engine.downloader.close() # cleanup
|
|
||||||
|
|
||||||
@coroutine_test
|
@coroutine_test
|
||||||
async def test_exception_slot(
|
async def test_no_spider(crawler: Crawler) -> None:
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
) -> None:
|
with pytest.raises(RuntimeError, match="Spider not opened"):
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
|
||||||
crawler.engine = engine
|
|
||||||
await engine.open_spider_async()
|
|
||||||
assert engine._slot
|
|
||||||
del engine._slot.heartbeat
|
|
||||||
await engine.close_spider_async()
|
await engine.close_spider_async()
|
||||||
assert "Slot close failure" in caplog.text
|
engine.downloader.close() # cleanup
|
||||||
|
|
||||||
@coroutine_test
|
|
||||||
async def test_exception_downloader(
|
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
|
||||||
) -> None:
|
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
|
||||||
crawler.engine = engine
|
|
||||||
await engine.open_spider_async()
|
|
||||||
del engine.downloader.slots
|
|
||||||
await engine.close_spider_async()
|
|
||||||
assert "Downloader close failure" in caplog.text
|
|
||||||
|
|
||||||
@coroutine_test
|
@coroutine_test
|
||||||
async def test_exception_scraper(
|
async def test_exception_slot(
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
) -> None:
|
) -> None:
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
crawler.engine = engine
|
crawler.engine = engine
|
||||||
await engine.open_spider_async()
|
await engine.open_spider_async()
|
||||||
engine.scraper.slot = None
|
assert engine._slot
|
||||||
await engine.close_spider_async()
|
del engine._slot.heartbeat
|
||||||
assert "Scraper close failure" in caplog.text
|
await engine.close_spider_async()
|
||||||
|
assert "Slot close failure" in caplog.text
|
||||||
|
|
||||||
@coroutine_test
|
|
||||||
async def test_exception_scheduler(
|
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
|
||||||
) -> None:
|
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
|
||||||
crawler.engine = engine
|
|
||||||
await engine.open_spider_async()
|
|
||||||
assert engine._slot
|
|
||||||
del cast("Scheduler", engine._slot.scheduler).dqs
|
|
||||||
await engine.close_spider_async()
|
|
||||||
assert "Scheduler close failure" in caplog.text
|
|
||||||
|
|
||||||
@coroutine_test
|
@coroutine_test
|
||||||
async def test_exception_signal(
|
async def test_exception_downloader(
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
) -> None:
|
) -> None:
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
crawler.engine = engine
|
crawler.engine = engine
|
||||||
await engine.open_spider_async()
|
await engine.open_spider_async()
|
||||||
signal_manager = engine.signals
|
del engine.downloader.slots
|
||||||
del engine.signals
|
await engine.close_spider_async()
|
||||||
await engine.close_spider_async()
|
assert "Downloader close failure" in caplog.text
|
||||||
assert "Error while sending spider_close signal" in caplog.text
|
|
||||||
# send the spider_closed signal to close various components
|
|
||||||
await signal_manager.send_catch_log_async(
|
|
||||||
signal=signals.spider_closed,
|
|
||||||
spider=engine.spider,
|
|
||||||
reason="cancelled",
|
|
||||||
)
|
|
||||||
|
|
||||||
@coroutine_test
|
|
||||||
async def test_exception_stats(
|
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
|
||||||
) -> None:
|
|
||||||
engine = ExecutionEngine(crawler, lambda _: None)
|
|
||||||
crawler.engine = engine
|
|
||||||
await engine.open_spider_async()
|
|
||||||
assert isinstance(crawler.stats, MemoryStatsCollector)
|
|
||||||
del crawler.stats.spider_stats
|
|
||||||
await engine.close_spider_async()
|
|
||||||
assert "Stats close failure" in caplog.text
|
|
||||||
|
|
||||||
@coroutine_test
|
@coroutine_test
|
||||||
async def test_exception_callback(
|
async def test_exception_scraper(
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
) -> None:
|
) -> None:
|
||||||
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
crawler.engine = engine
|
crawler.engine = engine
|
||||||
await engine.open_spider_async()
|
await engine.open_spider_async()
|
||||||
await engine.close_spider_async()
|
engine.scraper.slot = None
|
||||||
assert "Error running spider_closed_callback" in caplog.text
|
await engine.close_spider_async()
|
||||||
|
assert "Scraper close failure" in caplog.text
|
||||||
|
|
||||||
@coroutine_test
|
|
||||||
async def test_exception_async_callback(
|
|
||||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
|
||||||
) -> None:
|
|
||||||
async def cb(_):
|
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
engine = ExecutionEngine(crawler, cb)
|
@coroutine_test
|
||||||
crawler.engine = engine
|
async def test_exception_scheduler(
|
||||||
await engine.open_spider_async()
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
await engine.close_spider_async()
|
) -> None:
|
||||||
assert "Error running spider_closed_callback" in caplog.text
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
|
crawler.engine = engine
|
||||||
|
await engine.open_spider_async()
|
||||||
|
assert engine._slot
|
||||||
|
del cast("Scheduler", engine._slot.scheduler).dqs
|
||||||
|
await engine.close_spider_async()
|
||||||
|
assert "Scheduler close failure" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine_test
|
||||||
|
async def test_exception_signal(
|
||||||
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
|
crawler.engine = engine
|
||||||
|
await engine.open_spider_async()
|
||||||
|
signal_manager = engine.signals
|
||||||
|
del engine.signals
|
||||||
|
await engine.close_spider_async()
|
||||||
|
assert "Error while sending spider_close signal" in caplog.text
|
||||||
|
# send the spider_closed signal to close various components
|
||||||
|
await signal_manager.send_catch_log_async(
|
||||||
|
signal=signals.spider_closed,
|
||||||
|
spider=engine.spider,
|
||||||
|
reason="cancelled",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine_test
|
||||||
|
async def test_exception_stats(
|
||||||
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
engine = ExecutionEngine(crawler, lambda _: None)
|
||||||
|
crawler.engine = engine
|
||||||
|
await engine.open_spider_async()
|
||||||
|
assert isinstance(crawler.stats, MemoryStatsCollector)
|
||||||
|
del crawler.stats.spider_stats
|
||||||
|
await engine.close_spider_async()
|
||||||
|
assert "Stats close failure" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine_test
|
||||||
|
async def test_exception_callback(
|
||||||
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
|
||||||
|
crawler.engine = engine
|
||||||
|
await engine.open_spider_async()
|
||||||
|
await engine.close_spider_async()
|
||||||
|
assert "Error running spider_closed_callback" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine_test
|
||||||
|
async def test_exception_async_callback(
|
||||||
|
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
async def cb(_):
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
engine = ExecutionEngine(crawler, cb)
|
||||||
|
crawler.engine = engine
|
||||||
|
await engine.open_spider_async()
|
||||||
|
await engine.close_spider_async()
|
||||||
|
assert "Error running spider_closed_callback" in caplog.text
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue