Improve typing for Spider.parse(). (#6274)

.github/workflows/checks.yml

@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs

scrapy/commands/bench.py

@@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider):
         url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
-    def parse(self, response: Response) -> Any:  # type: ignore[override]
+    def parse(self, response: Response) -> Any:
         assert isinstance(Response, TextResponse)
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)

scrapy/spiders/__init__.py

@@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    # typing.Concatenate requires Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    from typing_extensions import Concatenate, Self
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
 
+    CallbackT = Callable[Concatenate[Response, ...], Any]
+
 
 class Spider(object_ref):
     """Base class for scrapy spiders. All spiders must inherit from this
@@ -79,10 +84,14 @@ class Spider(object_ref):
     def _parse(self, response: Response, **kwargs: Any) -> Any:
         return self.parse(response, **kwargs)
 
-    def parse(self, response: Response, **kwargs: Any) -> Any:
-        raise NotImplementedError(
-            f"{self.__class__.__name__}.parse callback is not defined"
-        )
+    if TYPE_CHECKING:
+        parse: CallbackT
+    else:
+
+        def parse(self, response: Response, **kwargs: Any) -> Any:
+            raise NotImplementedError(
+                f"{self.__class__.__name__}.parse callback is not defined"
+            )
 
     @classmethod
     def update_settings(cls, settings: BaseSettings) -> None:

tests_typing/test_spiders.mypy-testing

@@ -0,0 +1,68 @@
+from typing import Any
+
+import pytest
+
+from scrapy.http import HtmlResponse, Response
+from scrapy.spiders import Spider
+
+
+class SimpleSpider(Spider):
+    pass
+
+
+class SameOverrideSpider(Spider):
+    def parse(self, response: Response, **kwargs: Any) -> Any:
+        pass
+
+
+class NoKwargsSpider(Spider):
+    def parse(self, response: Response) -> Any:
+        pass
+
+
+class SpecificKwargsSpider(Spider):
+    def parse(self, response: Response, page: int) -> Any:
+        pass
+
+
+class NarrowOverrideSpider(Spider):
+    # without type: ignore this produces several note lines in addition to an error line,
+    # which is unsupported by pytest-mypy-testing
+    def parse(self, response: HtmlResponse, **kwargs: Any) -> Any:  # type: ignore[override]
+        pass
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse() -> None:
+    spider = Spider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_simple() -> None:
+    spider = SimpleSpider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_same() -> None:
+    spider = SameOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_no_kwargs() -> None:
+    spider = NoKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_specific_kwargs() -> None:
+    spider = SpecificKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_narrow() -> None:
+    spider = NarrowOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any

tox.ini

@@ -56,6 +56,15 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+basepython = python3.8
+deps =
+    {[test-requirements]deps}
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.3
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =