Pass the spider arg to custom stat collectors {open,close}_spider(). (#7213)

scrapy/core/engine.py

@@ -534,7 +534,16 @@ class ExecutionEngine:
             await maybe_deferred_to_future(d)
         await self.scraper.open_spider_async()
         assert self.crawler.stats
-        self.crawler.stats.open_spider()
+        if argument_is_required(self.crawler.stats.open_spider, "spider"):
+            warnings.warn(
+                f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+                f" this is deprecated and the argument will not be passed in future Scrapy versions.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self.crawler.stats.open_spider(spider=self.crawler.spider)
+        else:
+            self.crawler.stats.open_spider()
         await self.signals.send_catch_log_async(
             signals.spider_opened, spider=self.crawler.spider
         )
@@ -629,7 +638,18 @@ class ExecutionEngine:
 
         assert self.crawler.stats
         try:
-            self.crawler.stats.close_spider(reason=reason)
+            if argument_is_required(self.crawler.stats.close_spider, "spider"):
+                warnings.warn(
+                    f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+                    f" this is deprecated and the argument will not be passed in future Scrapy versions.",
+                    ScrapyDeprecationWarning,
+                    stacklevel=2,
+                )
+                self.crawler.stats.close_spider(
+                    spider=self.crawler.spider, reason=reason
+                )
+            else:
+                self.crawler.stats.close_spider(reason=reason)
         except Exception:
             log_failure("Stats close failure")
 

tests/test_stats.py

@@ -10,7 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
 from scrapy.statscollectors import DummyStatsCollector, StatsCollector
+from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler
+from tests.spiders import SimpleSpider
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
@@ -109,12 +111,77 @@ class TestStatsCollector:
         stats = StatsCollector(crawler)
         with pytest.warns(
             ScrapyDeprecationWarning,
-            match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated",
+            match=r"Passing a 'spider' argument to StatsCollector\.set_value\(\) is deprecated",
         ):
             stats.set_value("test", "value", spider=spider)
         assert stats.get_stats() == {"test": "value"}
         with pytest.warns(
             ScrapyDeprecationWarning,
-            match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated",
+            match=r"Passing a 'spider' argument to StatsCollector\.get_stats\(\) is deprecated",
         ):
             assert stats.get_stats(spider) == {"test": "value"}
+
+    @deferred_f_from_coro_f
+    async def test_deprecated_spider_arg_custom_collector(self) -> None:
+        class CustomStatsCollector:
+            def __init__(self, crawler):
+                self._stats = {}
+
+            def open_spider(self, spider):
+                pass
+
+            def get_stats(self, spider=None):
+                return self._stats
+
+            def inc_value(self, key, count=1, start=0, spider=None):
+                d = self._stats
+                d[key] = d.setdefault(key, start) + count
+
+            def close_spider(self, spider, reason):
+                pass
+
+        crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})
+        with (
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+            ),
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+            ),
+        ):
+            await crawler.crawl_async(url="data:,")
+
+    @deferred_f_from_coro_f
+    async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:
+        class CustomStatsCollector(StatsCollector):
+            def open_spider(self, spider):  # pylint: disable=signature-differs
+                super().open_spider(spider)
+
+            def inc_value(self, key, count=1, start=0, spider=None):  # pylint: disable=useless-parent-delegation
+                super().inc_value(key, count, start, spider)
+
+            def close_spider(self, spider, reason):  # pylint: disable=signature-differs
+                super().close_spider(spider, reason)
+
+        crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})
+        with (
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+            ),
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"Passing a 'spider' argument to .*CustomStatsCollector\.open_spider\(\) is deprecated",
+            ),
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+            ),
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"Passing a 'spider' argument to .*CustomStatsCollector\.close_spider\(\) is deprecated",
+            ),
+        ):
+            await crawler.crawl_async(url="data:,")