From dcf2483f8db76c668c08d040b85b9d73b3166e69 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 10 Aug 2026 17:41:08 +0200 Subject: [PATCH] killall unnecessary-asserts --- tests/benchmarks/__init__.py | 1 - tests/benchmarks/test_crawl.py | 5 ----- tests/test_closespider.py | 7 ------- tests/test_crawl.py | 4 ---- tests/test_crawler.py | 3 --- tests/test_downloadermiddleware_httpcache.py | 3 --- tests/test_downloadermiddleware_httpcompression.py | 2 -- tests/test_downloadermiddleware_offsite.py | 1 - tests/test_downloadermiddleware_retry.py | 12 ------------ tests/test_downloadermiddleware_stats.py | 3 --- tests/test_downloaderslotssettings.py | 1 - tests/test_dupefilters.py | 2 -- tests/test_engine_loop.py | 11 ----------- tests/test_extension_memdebug.py | 2 -- tests/test_extension_memusage.py | 3 --- tests/test_extension_telnet.py | 1 - tests/test_feedexport.py | 3 --- tests/test_feedexport_batch.py | 2 -- tests/test_logstats.py | 1 - tests/test_pipeline_crawl.py | 2 -- tests/test_pipeline_media.py | 1 - tests/test_request_cb_kwargs.py | 2 -- tests/test_scheduler.py | 3 --- tests/test_spider.py | 4 ---- tests/test_spider_start.py | 1 - tests/test_spidermiddleware_depth.py | 2 -- tests/test_spidermiddleware_httperror.py | 1 - tests/test_spidermiddleware_process_start.py | 1 - tests/test_spidermiddleware_urllength.py | 1 - tests/test_utils_log.py | 3 --- tests/test_utils_request.py | 5 ----- 31 files changed, 93 deletions(-) diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py index a066ed00a..7504ddf18 100644 --- a/tests/benchmarks/__init__.py +++ b/tests/benchmarks/__init__.py @@ -36,7 +36,6 @@ class NullDownloadHandler: async def download_request(self, request: Request) -> Response: self._active += 1 - assert self._crawler.stats self._crawler.stats.max_value("benchmark/peak_concurrency", self._active) try: await asyncio.sleep(0) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py index f79b66a5b..4f68e4f6f 100644 --- a/tests/benchmarks/test_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -128,7 +128,6 @@ class _DelayedPipeline: async def process_item(self, item: Any) -> Any: url = item["url"] self._active[url] += 1 - assert self._crawler.stats self._crawler.stats.max_value("benchmark/peak_items", self._active[url]) try: await asyncio.sleep(DELAY) @@ -147,7 +146,6 @@ def _crawl_tree( pages=pages, items=items, ) - assert crawler.stats assert crawler.stats.get_value("downloader/response_count") == domains * pages assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items return crawler @@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N def run() -> None: crawler = crawl(_FollowSpider, settings, url=url) - assert crawler.stats assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 benchmark(run) @@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None: def run() -> None: crawler = _crawl_tree({}, domains=1, pages=REQUESTS) - assert crawler.stats assert crawler.stats.get_value("benchmark/peak_concurrency") > 1 benchmark(run) @@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None: crawler = _crawl_tree( settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE ) - assert crawler.stats assert ( crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS ) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index dbb29a8bc..81c122b81 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -28,7 +28,6 @@ class TestCloseSpider: crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) assert isinstance(crawler.spider, ItemSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") @@ -40,7 +39,6 @@ class TestCloseSpider: crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) assert isinstance(crawler.spider, FollowAllSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") @@ -61,7 +59,6 @@ class TestCloseSpider: max_items=max_items, max_requests=max_requests, mockserver=self.mockserver ) assert isinstance(crawler.spider, MaxItemsAndRequestsSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -81,7 +78,6 @@ class TestCloseSpider: ) yield crawler.crawl(mockserver=self.mockserver) assert isinstance(crawler.spider, FollowAllSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -93,7 +89,6 @@ class TestCloseSpider: crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) assert isinstance(crawler.spider, ErrorSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" @@ -107,7 +102,6 @@ class TestCloseSpider: crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) assert isinstance(crawler.spider, FollowAllSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") @@ -119,7 +113,6 @@ class TestCloseSpider: crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) assert isinstance(crawler.spider, SlowSpider) - assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d284805be..b277526de 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -359,7 +359,6 @@ with multiples lines est: list[list[tuple[str, Any]]] = [] def cb(response): - assert crawler.engine est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -377,7 +376,6 @@ with multiples lines est: list[str] = [] def cb(response): - assert crawler.engine est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -452,7 +450,6 @@ class TestCrawlSpider: await crawler.crawl_async( mockserver.url("/status?n=200"), mockserver=mockserver ) - assert crawler.stats return items, crawler.stats @coroutine_test @@ -518,7 +515,6 @@ class TestCrawlSpider: # so the failure is dropped silently and the crawl finishes normally. assert "[parse] status 200 (foo: None)" in caplog.text assert "[errback]" not in caplog.text - assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/404") == 1 @coroutine_test diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 17ca02dea..2836a6627 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -536,7 +536,6 @@ class TestSpiderSettings: custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) - assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] assert AutoThrottle in enabled_exts @@ -568,7 +567,6 @@ class TestCrawlerLogging: } async def start(self): - assert crawler.stats info_count_start = crawler.stats.get_value("log_count/INFO") logging.debug("debug message") # noqa: LOG015 logging.info("info message") # noqa: LOG015 @@ -601,7 +599,6 @@ class TestCrawlerLogging: assert "info message" in logged assert "warning message" in logged assert "error message" in logged - assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 assert info_count == 1 diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index dc8228470..c90fba2bb 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -71,7 +71,6 @@ class TestBase: settings = self._get_settings(**new_settings) crawler = get_crawler(Spider, settings) crawler.spider = crawler._create_spider("example.com") - assert crawler.stats crawler.stats.open_spider() try: yield crawler @@ -136,7 +135,6 @@ class StorageTestMixin(TestBase): with self._middleware() as mw: spider = mw.crawler.spider assert spider - assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -158,7 +156,6 @@ class StorageTestMixin(TestBase): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: spider = mw.crawler.spider assert spider - assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a43bb51ba..1364c7cfd 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -60,7 +60,6 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) - assert self.crawler.stats self.crawler.stats.open_spider() def _getresponse(self, coding: str) -> Response: @@ -86,7 +85,6 @@ class TestHttpCompression: return response def assertStatsEqual(self, key: str, value: Any) -> None: - assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 1f91dcd4b..999efd8e6 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -229,7 +229,6 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen - assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index ab52590c7..908fb7957 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -81,7 +81,6 @@ class TestRetry: # discard it assert self.mw.process_response(req3, rsp) is rsp - assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -133,7 +132,6 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats - assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -314,7 +312,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -335,7 +332,6 @@ class TestGetRetryRequest: max_retry_times=max_retry_times, ) assert new_request is None - assert spider.crawler.stats assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" @@ -362,7 +358,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -395,7 +390,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -expected_retry_times expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) assert value == expected_retry_times @@ -520,7 +514,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -542,7 +535,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -568,7 +560,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -592,7 +583,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -618,7 +608,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -736,7 +725,6 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) - assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index b8fda25b4..90cfee15a 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -16,7 +16,6 @@ class MyException(Exception): class TestDownloaderStats: def setup_method(self) -> None: self.crawler = get_crawler(Spider) - assert self.crawler.stats is not None self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider() @@ -25,7 +24,6 @@ class TestDownloaderStats: self.res = Response("http://scrapytest.org", status=400) def assertStatsEqual(self, key: str, value: object) -> None: - assert self.crawler.stats is not None assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -52,7 +50,6 @@ class TestDownloaderStats: DownloaderStats.from_crawler(crawler) def teardown_method(self) -> None: - assert self.crawler.stats is not None self.crawler.stats.close_spider() diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d58a6e09..9f0a03b28 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -149,7 +149,6 @@ async def test_none_slot_with_priority_queue( assert crawler.spider.default_slot in crawler.spider.times assert len(crawler.spider.times[crawler.spider.default_slot]) == 2 - assert crawler.stats stats = crawler.stats assert stats.get_value("spider_exceptions", 0) == 0 assert stats.get_value("downloader/exception_count", 0) == 0 diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 479332eae..c2ca80669 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -178,7 +178,6 @@ class TestRFPDupeFilter: dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats assert crawler.stats.get_value("dupefilter/filtered") == 2 assert ( "scrapy.dupefilters", @@ -212,7 +211,6 @@ class TestRFPDupeFilter: dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats assert crawler.stats.get_value("dupefilter/filtered") == 2 assert ( "scrapy.dupefilters", diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 8f3484115..81217c9ba 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -65,7 +65,6 @@ class TestMain: name = "test" async def start(self) -> AsyncIterator[Any]: - assert self.crawler.engine assert self.crawler.engine._slot scheduler = self.crawler.engine._slot.scheduler assert isinstance(scheduler, MemoryScheduler) @@ -111,7 +110,6 @@ class TestMain: crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @@ -124,7 +122,6 @@ class TestMain: name = "test" async def start(self) -> AsyncIterator[Any]: - assert self.crawler.engine is not None await self.crawler.engine.close_async() yield Request("data:,a") @@ -145,7 +142,6 @@ class TestMain: await crawler.crawl_async() assert not caplog.records - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "shutdown" assert not actual_urls @@ -186,7 +182,6 @@ class TestMain: failure, response = errors[0] assert isinstance(failure.value, ValueError) assert response is None - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "start_error" assert crawler.stats.get_value("spider_exceptions/count") == 1 assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 @@ -213,7 +208,6 @@ class TestMain: await crawler.crawl_async() assert not caplog.records - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "my_reason" assert crawler.stats.get_value("spider_exceptions/count") is None @@ -296,7 +290,6 @@ class TestRequestSendOrder: crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_num, signals.request_reached_downloader) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" @@ -316,7 +309,6 @@ class TestRequestSendOrder: ) async def start(spider: Spider) -> AsyncIterator[Any]: - assert spider.crawler.engine assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. @@ -358,7 +350,6 @@ class TestRequestSendOrder: ) async def start(spider: Spider) -> AsyncIterator[Any]: - assert spider.crawler.engine assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. @@ -403,7 +394,6 @@ class TestRequestSendOrder: ) async def start(spider: Spider) -> AsyncIterator[Any]: - assert spider.crawler.engine assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. @@ -460,7 +450,6 @@ class TestRequestSendOrder: async def start(spider: Spider) -> AsyncIterator[Any]: for num in start_nums: - assert spider.crawler.engine if spider.crawler.engine.needs_backout(): await spider.crawler.signals.wait_for(signals.scheduler_empty) request = self.request(num, response_seconds, download_slots) diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py index e32989aec..25b427029 100644 --- a/tests/test_extension_memdebug.py +++ b/tests/test_extension_memdebug.py @@ -32,7 +32,6 @@ def test_spider_closed_sets_stats() -> None: ext.spider_closed(DefaultSpider(), "finished") - assert crawler.stats assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage) assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked) assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None @@ -46,6 +45,5 @@ async def test_crawl_sets_stats() -> None: crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True}) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1 diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index 76e8ca5d6..2cd9a9e63 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -98,7 +98,6 @@ async def test_memusage_below_thresholds_logs_peak( with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): await crawler.crawl_async(url="data:,", loops=1) - assert crawler.stats assert crawler.stats.get_value("memusage/limit_reached") is None assert crawler.stats.get_value("memusage/warning_reached") is None assert crawler.stats.get_value("memusage/max") == 25 * MB @@ -128,7 +127,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log( with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"): await crawler.crawl_async(url="data:,", loops=100) - assert crawler.stats assert crawler.stats.get_value("memusage/limit_reached") == 1 assert crawler.stats.get_value("finish_reason") == "memusage_exceeded" assert any( @@ -166,7 +164,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish( await crawler.crawl_async(url="data:,", loops=60) assert warning_signals == [1] - assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" warnings_logged = [ diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index cf858e4ea..41744c0b6 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -118,7 +118,6 @@ async def test_telnet_vars() -> None: start_urls = ["data:,"] async def parse(self, response: Response) -> None: - assert self.crawler.extensions console = next( ext for ext in self.crawler.extensions.middlewares diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d0f6a297c..ec57af629 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -309,7 +309,6 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @@ -333,7 +332,6 @@ class TestFeedExport(TestFeedExportBase): side_effect=store, ): yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 @@ -351,7 +349,6 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 4b0962c43..7265c136b 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -419,7 +419,6 @@ class TestBatchDeliveries(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(total=2, mockserver=self.mockserver) - assert crawler.stats assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @@ -490,7 +489,6 @@ class TestBatchDeliveries(TestFeedExportBase): assert len(CustomS3FeedStorage.stubs) == len(items) for stub in CustomS3FeedStorage.stubs: stub.assert_no_pending_responses() - assert crawler.stats assert ( "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() ) diff --git a/tests/test_logstats.py b/tests/test_logstats.py index 213681ad1..26c507b22 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -14,7 +14,6 @@ class TestLogStats: def setup_method(self) -> None: self.crawler = get_crawler(SimpleSpider) self.spider = self.crawler._create_spider("spidey") - assert self.crawler.stats is not None self.stats = self.crawler.stats self.stats.set_value("response_received_count", 4802) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 8b522255a..529e7ffcb 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -140,7 +140,6 @@ class TestFileDownloadCrawl: assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code - assert crawler.stats assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 assert crawler.stats.get_value("downloader/response_count") == 4 assert crawler.stats.get_value("downloader/response_status_count/200") == 1 @@ -208,7 +207,6 @@ class TestFileDownloadCrawl: mockserver=self.mockserver, ) self._assert_files_downloaded(self.items, caplog.text) - assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/302") == 3 @coroutine_test diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 50787d7b7..1032108d7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -81,7 +81,6 @@ class TestBaseMediaPipeline: self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo - assert crawler.request_fingerprinter is not None self.fingerprint = crawler.request_fingerprinter.fingerprint @property diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 6c26aa878..97dafc87c 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] def _inc_checks(self, count: int = 1) -> None: - assert self.crawler.stats self.crawler.stats.inc_value("boolean_checks", count) async def start(self): @@ -171,7 +170,6 @@ class TestCallbackKeywordArguments: await crawler.crawl_async(mockserver=mockserver) assert isinstance(crawler.spider, KeywordArgumentsSpider) assert all(crawler.spider.checks) - assert crawler.stats assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index d7c21bdbd..90582aac4 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -54,7 +54,6 @@ async def create_scheduler( finally: await ensure_awaitable(scheduler.close("finished")) await mock_crawler.stop_async() - assert mock_crawler.engine mock_crawler.engine.downloader.close() @@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase): dequeued_slots: list[str] = [] requests: list[Request] = [] assert scheduler.crawler - assert scheduler.crawler.engine downloader = scheduler.crawler.engine.downloader assert isinstance(downloader, MockDownloader) while scheduler.has_pending_requests(): @@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory: url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) - assert self.crawler.stats assert self.crawler.stats.get_value("downloader/response_count") == len( start_urls ) diff --git a/tests/test_spider.py b/tests/test_spider.py index 118b75422..6f7f171a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 @coroutine_test @@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1 @pytest.mark.parametrize("feed_iterator", ["xml", "html"]) @@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 @@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 7cef2fefa..fb72ece6c 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -34,7 +34,6 @@ class TestMain: crawler = get_crawler(spider) crawler.signals.connect(track_item, signals.item_scraped) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index bdacd7287..4818cffc5 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -26,7 +26,6 @@ def crawler() -> Crawler: @pytest.fixture def stats(crawler: Crawler) -> Generator[StatsCollector]: - assert crawler.stats is not None crawler.stats.open_spider() yield crawler.stats @@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None: Spider, {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10}, ) - assert crawler.stats is not None crawler.stats.open_spider() try: mw = build_from_crawler(DepthMiddleware, crawler) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 2ba083fd6..c0b2588f7 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -204,7 +204,6 @@ class TestHttpErrorMiddlewareIntegrational: assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert crawler.stats get_value = crawler.stats.get_value assert get_value("httperror/response_ignored_count") == 3 assert get_value("httperror/response_ignored_status_count/404") == 1 diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 491919759..46414bb59 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -80,7 +80,6 @@ class TestMain: crawler = get_crawler(spider_cls, settings_dict=settings) crawler.signals.connect(track_item, signals.item_scraped) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1ed3a5637..be27a9df5 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -31,7 +31,6 @@ def crawler() -> Crawler: @pytest.fixture def stats(crawler: Crawler) -> StatsCollector: - assert crawler.stats is not None return crawler.stats diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 42b2b95fd..71ae536d4 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -89,7 +89,6 @@ class TestLogCounterHandler: logger.removeHandler(handler) def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: - assert crawler.stats assert crawler.stats.get_value("log_count/DEBUG") is None assert crawler.stats.get_value("log_count/INFO") is None assert crawler.stats.get_value("log_count/WARNING") is None @@ -98,12 +97,10 @@ class TestLogCounterHandler: def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.error("test log msg") - assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.debug("test log msg") - assert crawler.stats assert crawler.stats.get_value("log_count/DEBUG") is None diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 935447bc4..6ca608053 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -274,7 +274,6 @@ class TestRequestFingerprinter: def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") - assert crawler.request_fingerprinter assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) @@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) @@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") @@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) @@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"]