From 0f07b2e38c0bbbef64d5bb35c6fe6ad504f2162f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 9 Jan 2026 19:57:53 +0400 Subject: [PATCH] Pass the spider arg to custom stat collectors {open,close}_spider(). (#7213) --- scrapy/core/engine.py | 24 +++++++++++++-- tests/test_stats.py | 71 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4f8d8e8a1..b29c5e80e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -534,7 +534,16 @@ class ExecutionEngine: await maybe_deferred_to_future(d) await self.scraper.open_spider_async() assert self.crawler.stats - self.crawler.stats.open_spider() + if argument_is_required(self.crawler.stats.open_spider, "spider"): + warnings.warn( + f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawler.stats.open_spider(spider=self.crawler.spider) + else: + self.crawler.stats.open_spider() await self.signals.send_catch_log_async( signals.spider_opened, spider=self.crawler.spider ) @@ -629,7 +638,18 @@ class ExecutionEngine: assert self.crawler.stats try: - self.crawler.stats.close_spider(reason=reason) + if argument_is_required(self.crawler.stats.close_spider, "spider"): + warnings.warn( + f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawler.stats.close_spider( + spider=self.crawler.spider, reason=reason + ) + else: + self.crawler.stats.close_spider(reason=reason) except Exception: log_failure("Stats close failure") diff --git a/tests/test_stats.py b/tests/test_stats.py index 6814a652e..b5ab06901 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -10,7 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider from scrapy.statscollectors import DummyStatsCollector, StatsCollector +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -109,12 +111,77 @@ class TestStatsCollector: stats = StatsCollector(crawler) with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated", + match=r"Passing a 'spider' argument to StatsCollector\.set_value\(\) is deprecated", ): stats.set_value("test", "value", spider=spider) assert stats.get_stats() == {"test": "value"} with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated", + match=r"Passing a 'spider' argument to StatsCollector\.get_stats\(\) is deprecated", ): assert stats.get_stats(spider) == {"test": "value"} + + @deferred_f_from_coro_f + async def test_deprecated_spider_arg_custom_collector(self) -> None: + class CustomStatsCollector: + def __init__(self, crawler): + self._stats = {} + + def open_spider(self, spider): + pass + + def get_stats(self, spider=None): + return self._stats + + def inc_value(self, key, count=1, start=0, spider=None): + d = self._stats + d[key] = d.setdefault(key, start) + count + + def close_spider(self, spider, reason): + pass + + crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + ): + await crawler.crawl_async(url="data:,") + + @deferred_f_from_coro_f + async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None: + class CustomStatsCollector(StatsCollector): + def open_spider(self, spider): # pylint: disable=signature-differs + super().open_spider(spider) + + def inc_value(self, key, count=1, start=0, spider=None): # pylint: disable=useless-parent-delegation + super().inc_value(key, count, start, spider) + + def close_spider(self, spider, reason): # pylint: disable=signature-differs + super().close_spider(spider, reason) + + crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to .*CustomStatsCollector\.open_spider\(\) is deprecated", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to .*CustomStatsCollector\.close_spider\(\) is deprecated", + ), + ): + await crawler.crawl_async(url="data:,")