From 93f06285309bd46e96fd147bf41e564c94b5bf2b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 13 May 2024 13:55:45 +0400
Subject: [PATCH] Improve typing for Spider.parse(). (#6274)

---
 .github/workflows/checks.yml           |  3 ++
 scrapy/commands/bench.py               |  2 +-
 scrapy/spiders/__init__.py             | 19 +++++--
 tests_typing/test_spiders.mypy-testing | 68 ++++++++++++++++++++++++++
 tox.ini                                |  9 ++++
 5 files changed, 95 insertions(+), 6 deletions(-)
 create mode 100644 tests_typing/test_spiders.mypy-testing

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d6fc0f6c5..ed1629b67 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 7523f3cfe..0c4ebcd23 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider):
         url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
-    def parse(self, response: Response) -> Any:  # type: ignore[override]
+    def parse(self, response: Response) -> Any:
         assert isinstance(Response, TextResponse)
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 72c2aaba7..2416d2a4d 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    # typing.Concatenate requires Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    from typing_extensions import Concatenate, Self
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
 
+    CallbackT = Callable[Concatenate[Response, ...], Any]
+
 
 class Spider(object_ref):
     """Base class for scrapy spiders. All spiders must inherit from this
@@ -79,10 +84,14 @@ class Spider(object_ref):
     def _parse(self, response: Response, **kwargs: Any) -> Any:
         return self.parse(response, **kwargs)
 
-    def parse(self, response: Response, **kwargs: Any) -> Any:
-        raise NotImplementedError(
-            f"{self.__class__.__name__}.parse callback is not defined"
-        )
+    if TYPE_CHECKING:
+        parse: CallbackT
+    else:
+
+        def parse(self, response: Response, **kwargs: Any) -> Any:
+            raise NotImplementedError(
+                f"{self.__class__.__name__}.parse callback is not defined"
+            )
 
     @classmethod
     def update_settings(cls, settings: BaseSettings) -> None:
diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing
new file mode 100644
index 000000000..162e31d0c
--- /dev/null
+++ b/tests_typing/test_spiders.mypy-testing
@@ -0,0 +1,68 @@
+from typing import Any
+
+import pytest
+
+from scrapy.http import HtmlResponse, Response
+from scrapy.spiders import Spider
+
+
+class SimpleSpider(Spider):
+    pass
+
+
+class SameOverrideSpider(Spider):
+    def parse(self, response: Response, **kwargs: Any) -> Any:
+        pass
+
+
+class NoKwargsSpider(Spider):
+    def parse(self, response: Response) -> Any:
+        pass
+
+
+class SpecificKwargsSpider(Spider):
+    def parse(self, response: Response, page: int) -> Any:
+        pass
+
+
+class NarrowOverrideSpider(Spider):
+    # without type: ignore this produces several note lines in addition to an error line,
+    # which is unsupported by pytest-mypy-testing
+    def parse(self, response: HtmlResponse, **kwargs: Any) -> Any:  # type: ignore[override]
+        pass
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse() -> None:
+    spider = Spider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_simple() -> None:
+    spider = SimpleSpider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_same() -> None:
+    spider = SameOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_no_kwargs() -> None:
+    spider = NoKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_specific_kwargs() -> None:
+    spider = SpecificKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_narrow() -> None:
+    spider = NarrowOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any
diff --git a/tox.ini b/tox.ini
index d7527bb04..ede139756 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,6 +56,15 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+basepython = python3.8
+deps =
+    {[test-requirements]deps}
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.3
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =