From 200d76afa96a78899faf9e2c30ef45273b71c600 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 1 Feb 2025 16:07:55 +0500 Subject: [PATCH] Refactor EngineTest tests. --- tests/test_engine.py | 82 +++++++++++----------- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 95955f7be..91ce2c0de 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,46 +243,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTest(unittest.TestCase): - @defer.inlineCallbacks - def test_crawler(self): - for spider in ( - TestSpider, - DictItemsSpider, - AttrsItemsSpider, - DataClassItemsSpider, - ): - run = CrawlerRun(spider) - yield run.run() - self._assert_visited_urls(run) - self._assert_scheduled_requests(run, count=9) - self._assert_downloaded_responses(run, count=9) - self._assert_scraped_items(run) - self._assert_signals_caught(run) - self._assert_bytes_received(run) - - @defer.inlineCallbacks - def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) - yield run.run() - self._assert_scheduled_requests(run, count=8) - self._assert_dropped_requests(run) - - @defer.inlineCallbacks - def test_crawler_itemerror(self): - run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield run.run() - self._assert_items_error(run) - - @defer.inlineCallbacks - def test_crawler_change_close_reason_on_idle(self): - run = CrawlerRun(ChangeCloseReasonSpider) - yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) - +class EngineTestBase(unittest.TestCase): def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = [ "/", @@ -422,6 +383,47 @@ class EngineTest(unittest.TestCase): run.signals_caught[signals.spider_closed], ) + +class EngineTest(EngineTestBase): + @defer.inlineCallbacks + def test_crawler(self): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): + run = CrawlerRun(spider) + yield run.run() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + + @defer.inlineCallbacks + def test_crawler_dupefilter(self): + run = CrawlerRun(TestDupeFilterSpider) + yield run.run() + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) + + @defer.inlineCallbacks + def test_crawler_itemerror(self): + run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield run.run() + self._assert_items_error(run) + + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + run = CrawlerRun(ChangeCloseReasonSpider) + yield run.run() + self.assertEqual( + {"spider": run.spider, "reason": "custom_reason"}, + run.signals_caught[signals.spider_closed], + ) + @defer.inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8dbb5b7ea..8bf225ab1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTest): +class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index db35bd81e..4efb6b7a8 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTest): +class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in (