diff --git a/docs/conf.py b/docs/conf.py index 1167ce050..493a62976 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,7 +3,6 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# pylint: disable=import-error import os import sys from collections.abc import Sequence diff --git a/pyproject.toml b/pyproject.toml index 871da8020..02ab7858d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,9 @@ extension-pkg-allow-list=[ enable = [ "useless-suppression", ] +# Make INFO checks like useless-suppression also cause pylint to return a +# non-zero exit code. +fail-on = "I" disable = [ # Ones we want to ignore "attribute-defined-outside-init", diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 171d8479c..f44f70e40 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy +import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast @@ -18,6 +19,8 @@ from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: @@ -95,9 +98,17 @@ class CrawlSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() + if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): + warnings.warn( + f"The CrawlSpider._parse_response method, which the " + f"{global_object_name(self.__class__)} class overrides, is " + f"deprecated: it will be removed in future Scrapy releases. " + f"Please override the CrawlSpider.parse_with_rules method " + f"instead." + ) def _parse(self, response: Response, **kwargs: Any) -> Any: - return self._parse_response( + return self.parse_with_rules( response=response, callback=self.parse_start_url, cb_kwargs=kwargs, @@ -137,7 +148,7 @@ class CrawlSpider(Spider): def _callback(self, response: Response, **cb_kwargs: Any) -> Any: rule = self._rules[cast(int, response.meta["rule"])] - return self._parse_response( + return self.parse_with_rules( response, cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, @@ -150,7 +161,7 @@ class CrawlSpider(Spider): failure, cast(Callable[[Failure], Any], rule.errback) ) - async def _parse_response( + async def parse_with_rules( self, response: Response, callback: CallbackT | None, @@ -171,6 +182,21 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> AsyncIterator[Any]: + warnings.warn( + "The CrawlSpider._parse_response method is deprecated: " + "it will be removed in future Scrapy releases. " + "Please use the CrawlSpider.parse_with_rules method instead.", + stacklevel=2, + ) + return self.parse_with_rules(response, callback, cb_kwargs, follow) + def _handle_failure( self, failure: Failure, errback: Callable[[Failure], Any] | None ) -> Iterable[Any]: diff --git a/tests/test_spider.py b/tests/test_spider.py index b4aa649a3..4e4a99638 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider): assert "Error while reading start items and requests" in str(log) assert "did you miss an 's'?" in str(log) + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + assert len(w) == 0 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + assert len(w) == 0 + spider = _CrawlSpider() + assert len(w) == 1 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 0 + class TestSitemapSpider(TestSpider): spider_class = SitemapSpider