Deprecate _parse_response and implement parse_with_rules (#6804)

docs/conf.py

@@ -3,7 +3,6 @@
 # For the full list of built-in configuration values, see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
-# pylint: disable=import-error
 import os
 import sys
 from collections.abc import Sequence

pyproject.toml

@@ -161,6 +161,9 @@ extension-pkg-allow-list=[
 enable = [
     "useless-suppression",
 ]
+# Make INFO checks like useless-suppression also cause pylint to return a
+# non-zero exit code.
+fail-on = "I"
 disable = [
     # Ones we want to ignore
     "attribute-defined-outside-init",

scrapy/spiders/crawl.py

@@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import copy
+import warnings
 from collections.abc import AsyncIterator, Awaitable, Callable
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
 
@@ -18,6 +19,8 @@ from scrapy.link import Link
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.deprecate import method_is_overridden
+from scrapy.utils.python import global_object_name
 from scrapy.utils.spider import iterate_spider_output
 
 if TYPE_CHECKING:
@@ -95,9 +98,17 @@ class CrawlSpider(Spider):
     def __init__(self, *a: Any, **kw: Any):
         super().__init__(*a, **kw)
         self._compile_rules()
+        if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"):
+            warnings.warn(
+                f"The CrawlSpider._parse_response method, which the "
+                f"{global_object_name(self.__class__)} class overrides, is "
+                f"deprecated: it will be removed in future Scrapy releases. "
+                f"Please override the CrawlSpider.parse_with_rules method "
+                f"instead."
+            )
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
-        return self._parse_response(
+        return self.parse_with_rules(
             response=response,
             callback=self.parse_start_url,
             cb_kwargs=kwargs,
@@ -137,7 +148,7 @@ class CrawlSpider(Spider):
 
     def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
         rule = self._rules[cast(int, response.meta["rule"])]
-        return self._parse_response(
+        return self.parse_with_rules(
             response,
             cast("CallbackT", rule.callback),
             {**rule.cb_kwargs, **cb_kwargs},
@@ -150,7 +161,7 @@ class CrawlSpider(Spider):
             failure, cast(Callable[[Failure], Any], rule.errback)
         )
 
-    async def _parse_response(
+    async def parse_with_rules(
         self,
         response: Response,
         callback: CallbackT | None,
@@ -171,6 +182,21 @@ class CrawlSpider(Spider):
             for request_or_item in self._requests_to_follow(response):
                 yield request_or_item
 
+    def _parse_response(
+        self,
+        response: Response,
+        callback: CallbackT | None,
+        cb_kwargs: dict[str, Any],
+        follow: bool = True,
+    ) -> AsyncIterator[Any]:
+        warnings.warn(
+            "The CrawlSpider._parse_response method is deprecated: "
+            "it will be removed in future Scrapy releases. "
+            "Please use the CrawlSpider.parse_with_rules method instead.",
+            stacklevel=2,
+        )
+        return self.parse_with_rules(response, callback, cb_kwargs, follow)
+
     def _handle_failure(
         self, failure: Failure, errback: Callable[[Failure], Any] | None
     ) -> Iterable[Any]:

tests/test_spider.py

@@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider):
         assert "Error while reading start items and requests" in str(log)
         assert "did you miss an 's'?" in str(log)
 
+    def test_parse_response_use(self):
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            start_urls = "https://www.example.com"
+            _follow_links = False
+
+        with warnings.catch_warnings(record=True) as w:
+            spider = _CrawlSpider()
+            assert len(w) == 0
+            spider._parse_response(
+                TextResponse(spider.start_urls, body=b""), None, None
+            )
+            assert len(w) == 1
+
+    def test_parse_response_override(self):
+        class _CrawlSpider(CrawlSpider):
+            def _parse_response(self, response, callback, cb_kwargs, follow=True):
+                pass
+
+            name = "test"
+            start_urls = "https://www.example.com"
+            _follow_links = False
+
+        with warnings.catch_warnings(record=True) as w:
+            assert len(w) == 0
+            spider = _CrawlSpider()
+            assert len(w) == 1
+            spider._parse_response(
+                TextResponse(spider.start_urls, body=b""), None, None
+            )
+            assert len(w) == 1
+
+    def test_parse_with_rules(self):
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            start_urls = "https://www.example.com"
+
+        with warnings.catch_warnings(record=True) as w:
+            spider = _CrawlSpider()
+            spider.parse_with_rules(
+                TextResponse(spider.start_urls, body=b""), None, None
+            )
+            assert len(w) == 0
+
 
 class TestSitemapSpider(TestSpider):
     spider_class = SitemapSpider