More granular untyped defs checking in tests. (#7712)

pyproject.toml

@@ -94,7 +94,82 @@ untyped_calls_exclude = [
 [[tool.mypy.overrides]]
 module = "tests.*"
 allow_untyped_defs = true
-allow_incomplete_defs = true # 48 errors
+allow_incomplete_defs = true # 59 errors
+
+# TODO
+[[tool.mypy.overrides]]
+module = [
+    "tests.mockserver.*",
+    "tests.spiders",
+    "tests.test_closespider",
+    "tests.test_cmdline",
+    "tests.test_contracts",
+    "tests.test_core_downloader",
+    "tests.test_downloader_handler_twisted_ftp",
+    "tests.test_downloadermiddleware_cookies",
+    "tests.test_downloadermiddleware_httpauth",
+    "tests.test_downloadermiddleware_httpcache",
+    "tests.test_downloadermiddleware_httpcompression",
+    "tests.test_downloadermiddleware_httpproxy",
+    "tests.test_downloadermiddleware_offsite",
+    "tests.test_downloadermiddleware_redirect",
+    "tests.test_downloadermiddleware_redirect_base",
+    "tests.test_downloadermiddleware_redirect_metarefresh",
+    "tests.test_downloadermiddleware_retry",
+    "tests.test_downloadermiddleware_robotstxt",
+    "tests.test_downloadermiddleware_stats",
+    "tests.test_downloaderslotssettings",
+    "tests.test_dupefilters",
+    "tests.test_engine_loop",
+    "tests.test_exporters",
+    "tests.test_extension_statsmailer",
+    "tests.test_extension_throttle",
+    "tests.test_feedexport",
+    "tests.test_feedexport_postprocess",
+    "tests.test_feedexport_storages",
+    "tests.test_feedexport_uri_params",
+    "tests.test_http2_client_protocol",
+    "tests.test_http_headers",
+    "tests.test_http_request",
+    "tests.test_http_request_form",
+    "tests.test_http_response",
+    "tests.test_http_response_text",
+    "tests.test_item",
+    "tests.test_link",
+    "tests.test_linkextractors",
+    "tests.test_loader",
+    "tests.test_logformatter",
+    "tests.test_logstats",
+    "tests.test_mail",
+    "tests.test_pipeline_crawl",
+    "tests.test_pipeline_files",
+    "tests.test_pipeline_images",
+    "tests.test_pipeline_media",
+    "tests.test_pipelines",
+    "tests.test_pqueues",
+    "tests.test_request_attribute_binding",
+    "tests.test_request_cb_kwargs",
+    "tests.test_request_dict",
+    "tests.test_request_left",
+    "tests.test_robotstxt_interface",
+    "tests.test_scheduler_base",
+    "tests.test_settings",
+    "tests.test_spider",
+    "tests.test_spider_crawl",
+    "tests.test_spidermiddleware_output_chain",
+    "tests.test_spidermiddleware_process_start",
+    "tests.test_spider_sitemap",
+    "tests.test_squeues",
+    "tests.test_squeues_request",
+    "tests.test_stats",
+    "tests.test_utils_datatypes",
+    "tests.test_utils_decorators",
+    "tests.test_utils_defer",
+    "tests.test_utils_deprecate",
+    "tests.test_utils_misc.test_return_with_argument_inside_generator",
+    "tests.test_utils_python",
+    "tests.test_utils_request",
+]
 check_untyped_defs = false
 
 # Interface classes are hard to support

tests/AsyncCrawlerProcess/asyncio_deferred_signal.py

@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
 
 
 class UppercasePipeline:
-    async def _open_spider(self, spider):
+    async def _open_spider(self, spider: Spider) -> None:
         spider.logger.info("async pipeline opened!")
         await asyncio.sleep(0.1)
 

tests/AsyncCrawlerProcess/caching_hostname_resolver.py

@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
+    url: str
 
     async def start(self):
         yield scrapy.Request(self.url)

tests/CrawlerProcess/asyncio_deferred_signal.py

@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
 
 
 class UppercasePipeline:
-    async def _open_spider(self, spider):
+    async def _open_spider(self, spider: Spider) -> None:
         spider.logger.info("async pipeline opened!")
         await asyncio.sleep(0.1)
 

tests/CrawlerProcess/caching_hostname_resolver.py

@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
+    url: str
 
     async def start(self):
         yield scrapy.Request(self.url)

tests/CrawlerProcess/sleeping.py

@@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider):
     async def parse(self, response):
         from twisted.internet import reactor
 
-        d = Deferred()
+        d: Deferred[None] = Deferred()
         reactor.callLater(int(sys.argv[1]), d.callback, None)
         await maybe_deferred_to_future(d)
 

tests/CrawlerRunner/ip_address.py

@@ -36,6 +36,7 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:
 
 class LocalhostSpider(Spider):
     name = "localhost_spider"
+    url: str
 
     async def start(self):
         yield Request(self.url)

tests/test_addons.py

@@ -107,11 +107,11 @@ class TestAddonManager:
         addonlist = []
         for i in range(3):
             addon = get_addon_cls({"KEY1": i})
-            addon.number = i
+            addon.number = i  # type: ignore[attr-defined]
             addonlist.append(addon)
         # Test for every possible ordering
         for ordered_addons in itertools.permutations(addonlist):
-            expected_order = [a.number for a in ordered_addons]
+            expected_order = [a.number for a in ordered_addons]  # type: ignore[attr-defined]
             settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
             crawler = get_crawler(settings_dict=settings)
             manager = crawler.addons
@@ -177,7 +177,7 @@ class TestAddonManager:
                     )
                 settings["SCHEDULER"] = "AddonScheduler"
 
-        settings_dict = {
+        settings_dict: dict[str, Any] = {
             "ADDONS": {AddonWithFallback: 1},
         }
         crawler = get_crawler(settings_dict=settings_dict)

tests/test_cmdline_crawl_with_pipeline/__init__.py

@@ -1,21 +1,24 @@
+from __future__ import annotations
+
 import sys
 from pathlib import Path
 from subprocess import PIPE, Popen
 
 
-class TestCmdlineCrawlPipeline:
+def _execute(spname: str) -> int:
-    def _execute(self, spname):
+    args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
-        args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
+    cwd = Path(__file__).resolve().parent
-        cwd = Path(__file__).resolve().parent
+    proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
-        proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
+    proc.communicate()
-        _, stderr = proc.communicate()
+    return proc.returncode
-        return proc.returncode, stderr
 
-    def test_open_spider_normally_in_pipeline(self):
-        returncode, _ = self._execute("normal")
-        assert returncode == 0
 
-    def test_exception_at_open_spider_in_pipeline(self):
+def test_open_spider_normally_in_pipeline():
-        returncode, _ = self._execute("exception")
+    returncode = _execute("normal")
-        # An exception in pipeline's open_spider should result in a non-zero exit code
+    assert returncode == 0
-        assert returncode == 1
+
+
+def test_exception_at_open_spider_in_pipeline():
+    returncode = _execute("exception")
+    # An exception in pipeline's open_spider should result in a non-zero exit code
+    assert returncode == 1

tests/test_commands.py

@@ -89,6 +89,7 @@ class TestCommandSettings:
             args=["-s", f"FEEDS={feeds_json}", "spider.py"]
         )
         self.command.process_options(args, opts)
+        assert self.command.settings is not None
         assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings)
         assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json)
 

tests/test_crawl.py

@@ -356,9 +356,10 @@ with multiples lines
 
     @coroutine_test
     async def test_engine_status(self, mockserver: MockServer) -> None:
-        est = []
+        est: list[list[tuple[str, Any]]] = []
 
         def cb(response):
+            assert crawler.engine
             est.append(get_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
@@ -373,9 +374,10 @@ with multiples lines
 
     @coroutine_test
     async def test_format_engine_status(self, mockserver: MockServer) -> None:
-        est = []
+        est: list[str] = []
 
         def cb(response):
+            assert crawler.engine
             est.append(format_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
@@ -386,8 +388,8 @@ with multiples lines
         assert len(est) == 1, est
         est = est[0].split("\n")[2:-2]  # remove header & footer
         # convert to dict
-        est = [x.split(":") for x in est]
+        est_split = [x.split(":") for x in est]
-        est = [x for sublist in est for x in sublist]  # flatten
+        est = [x for sublist in est_split for x in sublist]  # flatten
         est = [x.lstrip().rstrip() for x in est]
         it = iter(est)
         s = dict(zip(it, it, strict=True))

tests/test_crawler.py

@@ -523,6 +523,7 @@ class TestCrawlerLogging:
             }
 
             async def start(self):
+                assert crawler.stats
                 info_count_start = crawler.stats.get_value("log_count/INFO")
                 logging.debug("debug message")  # noqa: LOG015
                 logging.info("info message")  # noqa: LOG015

tests/test_dependencies.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import re
 from configparser import ConfigParser
@@ -23,6 +25,7 @@ class TestScrapyUtils:
         config_parser.read(tox_config_file_path)
         pattern = r"Twisted==([\d.]+)"
         match = re.search(pattern, config_parser["min"]["deps"])
+        assert match
         pinned_twisted_version_string = match[1]
 
         assert twisted_version.short() == pinned_twisted_version_string

tests/test_downloader_handlers.py

@@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Request
+from scrapy.http import Request, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.misc import build_from_crawler
@@ -354,6 +354,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "A brief note"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "US-ASCII"
 
     @coroutine_test
@@ -362,6 +363,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "iso-8859-7"
 
     @coroutine_test
@@ -370,6 +372,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert response.body == b"\xbe\xd3\xbe"
+        assert isinstance(response, TextResponse)
         assert response.encoding == "iso-8859-7"
 
     @coroutine_test
@@ -382,6 +385,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "utf-8"
 
     @coroutine_test

tests/test_downloadermiddleware.py

@@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
                 return result
 
             def process_request(self, request):
-                d = Deferred()
+                d: Deferred[Response] = Deferred()
                 d.addCallback(self.cb)
                 d.callback(resp)
                 return d
@@ -298,6 +298,8 @@ class TestDownloadDeprecated(TestManagerBase):
             return succeed(resp)
 
         async with self.get_mwman() as mwman:
+            assert mwman.crawler
+            assert mwman.crawler.spider
             with pytest.warns(
                 ScrapyDeprecationWarning,
                 match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",

tests/test_downloadermiddleware_defaultheaders.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
@@ -5,28 +7,29 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 
 
-class TestDefaultHeadersMiddleware:
+def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]:
-    def get_defaults_mw(self):
+    crawler = get_crawler(Spider)
-        crawler = get_crawler(Spider)
+    defaults = {
-        defaults = {
+        to_bytes(k): [to_bytes(v)]
-            to_bytes(k): [to_bytes(v)]
+        for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
-            for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
+    }
-        }
+    return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
-        return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
 
-    def test_process_request(self):
-        defaults, mw = self.get_defaults_mw()
-        req = Request("http://www.scrapytest.org")
-        mw.process_request(req)
-        assert req.headers == defaults
 
-    def test_update_headers(self):
+def test_process_request():
-        defaults, mw = self.get_defaults_mw()
+    defaults, mw = get_defaults_mw()
-        headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
+    req = Request("http://www.scrapytest.org")
-        bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
+    mw.process_request(req)
-        req = Request("http://www.scrapytest.org", headers=headers)
+    assert req.headers == defaults
-        assert req.headers == bytes_headers
 
-        mw.process_request(req)
+
-        defaults.update(bytes_headers)
+def test_update_headers():
-        assert req.headers == defaults
+    defaults, mw = get_defaults_mw()
+    headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
+    bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
+    req = Request("http://www.scrapytest.org", headers=headers)
+    assert req.headers == bytes_headers
+
+    mw.process_request(req)
+    defaults.update(bytes_headers)
+    assert req.headers == defaults

tests/test_downloadermiddleware_downloadtimeout.py

@@ -1,43 +1,51 @@
+from __future__ import annotations
+
+from typing import Any
+
 from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
-class TestDownloadTimeoutMiddleware:
+def get_request_spider_mw(settings: dict[str, Any] | None = None):
-    def get_request_spider_mw(self, settings=None):
+    crawler = get_crawler(Spider, settings)
-        crawler = get_crawler(Spider, settings)
+    spider = crawler._create_spider("foo")
-        spider = crawler._create_spider("foo")
+    request = Request("http://scrapytest.org/")
-        request = Request("http://scrapytest.org/")
+    return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
-        return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
 
-    def test_default_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw()
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 180
 
-    def test_string_download_timeout(self):
+def test_default_download_timeout():
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
+    req, spider, mw = get_request_spider_mw()
-        mw.spider_opened(spider)
+    mw.spider_opened(spider)
-        assert mw.process_request(req) is None
+    assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 20.1
+    assert req.meta.get("download_timeout") == 180
 
-    def test_setting_has_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 2
 
-    def test_request_has_download_timeout(self):
+def test_string_download_timeout():
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
-        mw.spider_opened(spider)
+    mw.spider_opened(spider)
-        req.meta["download_timeout"] = 1
+    assert mw.process_request(req) is None
-        assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 20.1
-        assert req.meta.get("download_timeout") == 1
 
-    def test_zero_download_timeout(self):
+
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
+def test_setting_has_download_timeout():
-        mw.spider_opened(spider)
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
-        assert mw.process_request(req) is None
+    mw.spider_opened(spider)
-        assert req.meta.get("download_timeout") is None
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 2
+
+
+def test_request_has_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+    mw.spider_opened(spider)
+    req.meta["download_timeout"] = 1
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 1
+
+
+def test_zero_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
+    mw.spider_opened(spider)
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") is None

tests/test_downloadermiddleware_useragent.py

@@ -1,33 +1,37 @@
+from __future__ import annotations
+
 from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
-class TestUserAgentMiddleware:
+def get_spider_and_mw(
-    def get_spider_and_mw(self, default_useragent):
+    default_useragent: str | None,
-        crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
+) -> tuple[Spider, UserAgentMiddleware]:
-        spider = crawler._create_spider("foo")
+    crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
-        return spider, UserAgentMiddleware.from_crawler(crawler)
+    spider = crawler._create_spider("foo")
+    return spider, UserAgentMiddleware.from_crawler(crawler)
 
-    def test_default_agent(self):
-        _, mw = self.get_spider_and_mw("default_useragent")
-        req = Request("http://scrapytest.org/")
-        assert mw.process_request(req) is None
-        assert req.headers["User-Agent"] == b"default_useragent"
 
-    def test_header_agent(self):
+def test_default_agent():
-        spider, mw = self.get_spider_and_mw("default_useragent")
+    _, mw = get_spider_and_mw("default_useragent")
-        mw.spider_opened(spider)
+    req = Request("http://scrapytest.org/")
-        req = Request(
+    assert mw.process_request(req) is None
-            "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
+    assert req.headers["User-Agent"] == b"default_useragent"
-        )
-        assert mw.process_request(req) is None
-        assert req.headers["User-Agent"] == b"header_useragent"
 
-    def test_no_agent(self):
+
-        spider, mw = self.get_spider_and_mw(None)
+def test_header_agent():
-        mw.spider_opened(spider)
+    spider, mw = get_spider_and_mw("default_useragent")
-        req = Request("http://scrapytest.org/")
+    mw.spider_opened(spider)
-        assert mw.process_request(req) is None
+    req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})
-        assert "User-Agent" not in req.headers
+    assert mw.process_request(req) is None
+    assert req.headers["User-Agent"] == b"header_useragent"
+
+
+def test_no_agent():
+    spider, mw = get_spider_and_mw(None)
+    mw.spider_opened(spider)
+    req = Request("http://scrapytest.org/")
+    assert mw.process_request(req) is None
+    assert "User-Agent" not in req.headers

tests/test_engine.py

@@ -38,6 +38,8 @@ from tests import get_testdata
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from twisted.python.failure import Failure
 
     from scrapy.core.scheduler import Scheduler
@@ -608,8 +610,8 @@ class TestEngineDownload(TestEngineDownloadAsync):
 @coroutine_test
 async def test_request_scheduled_signal():
     class TestScheduler(BaseScheduler):
-        def __init__(self):
+        def __init__(self) -> None:
-            self.enqueued = []
+            self.enqueued: list[Request] = []
 
         def enqueue_request(self, request: Request) -> bool:
             self.enqueued.append(request)
@@ -621,9 +623,9 @@ async def test_request_scheduled_signal():
 
     crawler = get_crawler(MySpider)
     engine = ExecutionEngine(crawler, lambda _: None)
-    scheduler = TestScheduler()
+    scheduler = TestScheduler()  # type: ignore[abstract]
 
-    async def start():
+    async def start() -> AsyncIterator[Any]:
         return
         yield
 

tests/test_extension_periodic_log.py

@@ -102,7 +102,9 @@ class TestPeriodicLog:
             ext.spider_closed(spider, reason="finished")
             return ext, a, b
 
-        def check(settings: dict[str, Any], condition: Callable) -> None:
+        def check(
+            settings: dict[str, Any], condition: Callable[[str, Any], bool]
+        ) -> None:
             ext, a, b = emulate(settings)
             assert list(a["delta"].keys()) == [
                 k for k, v in ext.stats._stats.items() if condition(k, v)
@@ -168,7 +170,9 @@ class TestPeriodicLog:
             ext.spider_closed(spider, reason="finished")
             return ext, a, b
 
-        def check(settings: dict[str, Any], condition: Callable) -> None:
+        def check(
+            settings: dict[str, Any], condition: Callable[[str, Any], bool]
+        ) -> None:
             ext, a, b = emulate(settings)
             assert list(a["stats"].keys()) == [
                 k for k, v in ext.stats._stats.items() if condition(k, v)

tests/test_feedexport_batch.py

@@ -378,6 +378,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         crawler = get_crawler(ItemSpider, settings)
         yield crawler.crawl(total=2, mockserver=self.mockserver)
+        assert crawler.stats
         assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
         assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
 
@@ -392,7 +393,9 @@ class TestBatchDeliveries(TestFeedExportBase):
         ]
 
         class CustomS3FeedStorage(S3FeedStorage):
-            stubs = []
+            from botocore.stub import Stubber  # noqa: PLC0415
+
+            stubs: list[Stubber] = []
 
             def open(self, *args, **kwargs):
                 from botocore import __version__ as botocore_version  # noqa: PLC0415
@@ -449,6 +452,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         assert len(CustomS3FeedStorage.stubs) == len(items)
         for stub in CustomS3FeedStorage.stubs:
             stub.assert_no_pending_responses()
+        assert crawler.stats
         assert (
             "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
         )

tests/test_http_cookies.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from http.cookiejar import DefaultCookiePolicy
 
 from scrapy.http import Request, Response
@@ -46,7 +48,7 @@ class TestCookieJar:
     def test_set_policy(self):
         policy = DefaultCookiePolicy()
         self.jar.set_policy(policy)
-        assert self.jar.jar._policy is policy
+        assert self.jar.jar._policy is policy  # type: ignore[attr-defined]
 
     def test_check_expired_frequency(self):
         jar = CookieJar(check_expired_frequency=1)

tests/test_responsetypes.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from scrapy.http import (
     Headers,
     HtmlResponse,
@@ -38,13 +40,13 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_content_disposition_no_filename(self):
         assert responsetypes.from_content_disposition(b"attachment") is Response
 
     def test_from_content_type(self):
-        mappings = [
+        mappings: list[tuple[str | bytes, type[Response]]] = [
             ("text/html; charset=UTF-8", HtmlResponse),
             ("text/xml; charset=UTF-8", XmlResponse),
             ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
@@ -58,7 +60,7 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_body(self):
         mappings = [
@@ -71,7 +73,7 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_body(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_headers(self):
         mappings = [
@@ -98,7 +100,7 @@ class TestResponseTypes:
 
     def test_from_args(self):
         # TODO: add more tests that check precedence between the different arguments
-        mappings = [
+        mappings: list[tuple[dict[str, Any], type[Response]]] = [
             ({"url": "http://www.example.com/data.csv"}, TextResponse),
             # headers takes precedence over url
             (

tests/test_scheduler.py

@@ -379,13 +379,14 @@ class TestIntegrationWithDownloaderAwareInMemory:
             url = mockserver.url("/status?n=200", is_secure=False)
             start_urls = [url] * 6
             yield self.crawler.crawl(start_urls)
+            assert self.crawler.stats
             assert self.crawler.stats.get_value("downloader/response_count") == len(
                 start_urls
             )
 
 
 class TestIncompatibility:
-    def _incompatible(self):
+    def _incompatible(self) -> None:
         settings = {
             "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
             "CONCURRENT_REQUESTS_PER_IP": 1,

tests/test_signals.py

@@ -1,13 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import pytest
 
 from scrapy import Request, Spider, signals
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.mockserver.http import MockServer
+from tests.utils.decorators import coroutine_test
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 
 class ItemSpider(Spider):
     name = "itemspider"
+    mockserver: MockServer
 
     async def start(self):
         for index in range(10):
@@ -34,15 +41,6 @@ class TestMain:
 
 
 class TestMockServer:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
     def setup_method(self):
         self.items = []
 
@@ -51,11 +49,11 @@ class TestMockServer:
         self.items.append(item)
 
     @pytest.mark.only_asyncio
-    @inline_callbacks_test
+    @coroutine_test
-    def test_simple_pipeline(self):
+    async def test_simple_pipeline(self, mockserver: MockServer) -> None:
         crawler = get_crawler(ItemSpider)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
         assert len(self.items) == 10
         for index in range(10):
             assert {"index": index} in self.items

tests/test_spider_start.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from asyncio import sleep
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
@@ -12,6 +12,9 @@ from scrapy.utils.test import get_crawler
 from .utils import twisted_sleep
 from .utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Callable
+
 SLEEP_SECONDS = 0.1
 
 ITEM_A = {"id": "a"}
@@ -67,7 +70,11 @@ class TestMain:
 
         await self._test_spider(TestSpider, [ITEM_A])
 
-    async def _test_start(self, start_, expected_items=None):
+    async def _test_start(
+        self,
+        start_: Callable[[Any], AsyncIterator[Any]],
+        expected_items: list[Any] | None = None,
+    ) -> None:
         class TestSpider(Spider):
             name = "test"
             start = start_

tests/test_spidermiddleware.py

@@ -186,7 +186,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware:
 class ProcessSpiderExceptionAsyncIteratorMiddleware:
     async def process_spider_exception(self, response, exception):
         yield {"foo": 1}
-        d = defer.Deferred()
+        d: defer.Deferred[None] = defer.Deferred()
         call_later(0, d.callback, None)
         await maybe_deferred_to_future(d)
         yield {"foo": 2}

tests/test_spidermiddleware_httperror.py

@@ -22,6 +22,7 @@ class _HttpErrorSpider(MockServerSpider):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        assert self.mockserver
         self.start_urls = [
             self.mockserver.url("/status?n=200"),
             self.mockserver.url("/status?n=404"),
@@ -36,7 +37,7 @@ class _HttpErrorSpider(MockServerSpider):
         for url in self.start_urls:
             yield Request(url, self.parse, errback=self.on_error)
 
-    def parse(self, response):
+    def parse(self, response: Response) -> None:
         self.parsed.add(response.url[-3:])
 
     def on_error(self, failure):

tests/test_spidermiddleware_referer.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 import pytest
@@ -35,6 +35,9 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
 
 class TestRefererMiddleware:
     req_meta: dict[str, Any] = {}
@@ -1031,7 +1034,7 @@ async def test_response_policy_only_supports_policy_names():
     crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
     mw = build_from_crawler(RefererMiddleware, crawler)
 
-    async def input_result():
+    async def input_result() -> AsyncIterator[Any]:
         yield Request("https://example.com/")
 
     response = Response(
@@ -1079,7 +1082,7 @@ async def test_referer_policies_setting():
     )
     mw = build_from_crawler(RefererMiddleware, crawler)
 
-    async def input_result():
+    async def input_result() -> AsyncIterator[Any]:
         yield Request("https://example.com/")
 
     # "no-referrer-when-downgrade": None,

tests/test_spidermiddleware_start.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.start import StartSpiderMiddleware
 from scrapy.spiders import Spider
@@ -5,6 +9,9 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
 
 class TestMiddleware:
     @coroutine_test
@@ -12,7 +19,7 @@ class TestMiddleware:
         crawler = get_crawler(Spider)
         mw = build_from_crawler(StartSpiderMiddleware, crawler)
 
-        async def start():
+        async def start() -> AsyncIterator[Request]:
             yield Request("data:,1")
             yield Request("data:,2", meta={"is_start_request": True})
             yield Request("data:,2", meta={"is_start_request": False})

tests/test_utils_asyncio.py

@@ -72,7 +72,7 @@ class TestParallelAsyncio:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable(length)
             await _parallel_asyncio(
                 ait,
@@ -91,7 +91,7 @@ class TestParallelAsyncio:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable_with_delays(length)
             await _parallel_asyncio(
                 ait,

tests/test_utils_conf.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+
 import pytest
 
 from scrapy.exceptions import UsageError
@@ -106,7 +110,7 @@ class TestFeedExportConfig:
             )
 
     def test_feed_complete_default_values_from_settings_empty(self):
-        feed = {}
+        feed: dict[str, Any] = {}
         settings = Settings(
             {
                 "FEED_EXPORT_ENCODING": "custom encoding",

tests/test_utils_console.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.utils.console import get_shell_embed_func
@@ -38,4 +40,5 @@ def test_get_shell_embed_func_bpython():
 def test_get_shell_embed_func_ipython():
     # default shell should be 'ipython'
     shell = get_shell_embed_func()
+    assert shell is not None
     assert shell.__name__ == "_embed_ipython_shell"

tests/test_utils_display.py

@@ -1,9 +1,16 @@
+from __future__ import annotations
+
 import builtins
 from io import StringIO
+from typing import TYPE_CHECKING
 from unittest import mock
 
 from scrapy.utils.display import pformat, pprint
 
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from types import ModuleType
+
 value = {"a": 1}
 colorized_strings = {
     (
@@ -77,14 +84,19 @@ def test_pformat_no_pygments(isatty):
 
     real_import = builtins.__import__
 
-    def mock_import(name, globals_, locals_, fromlist, level):
+    def mock_import(
+        name: str,
+        globals_: Mapping[str, object] | None = None,
+        locals_: Mapping[str, object] | None = None,
+        fromlist: Sequence[str] | None = (),
+        level: int = 0,
+    ) -> ModuleType:
         if "pygments" in name:
             raise ImportError
         return real_import(name, globals_, locals_, fromlist, level)
 
-    builtins.__import__ = mock_import
+    with mock.patch("builtins.__import__", mock_import):
-    assert pformat(value) == plain_string
+        assert pformat(value) == plain_string
-    builtins.__import__ = real_import
 
 
 def test_pprint():

tests/test_utils_iterators.py

@@ -238,7 +238,7 @@ class TestXmliterBase(ABC):
             next(my_iter)
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")
+        i = self.xmliter(42, "product")  # type: ignore[arg-type]
         with pytest.raises(TypeError):
             next(i)
 
@@ -353,7 +353,7 @@ class TestLxmlXmliter(TestXmliterBase):
         assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")
+        i = self.xmliter(42, "product")  # type: ignore[arg-type]
         with pytest.raises(TypeError):
             next(i)
 

tests/test_utils_log.py

@@ -38,7 +38,7 @@ class TestFailureToExcInfo:
         assert exc_info == failure_to_exc_info(failure)
 
     def test_non_failure(self):
-        assert failure_to_exc_info("test") is None
+        assert failure_to_exc_info("test") is None  # type: ignore[arg-type]
 
 
 class TestTopLevelFormatter:

tests/test_utils_misc/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import sys
 from pathlib import Path
@@ -37,7 +39,7 @@ class TestUtilsMisc:
         with pytest.raises(NameError):
             load_object("scrapy.utils.misc.load_object999")
         with pytest.raises(TypeError):
-            load_object({})
+            load_object({})  # type: ignore[arg-type]
 
     def test_walk_modules(self):
         mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
@@ -114,7 +116,7 @@ class TestUtilsMisc:
         args = (True, 100.0)
         kwargs = {"key": "val"}
 
-        def _test_with_crawler(mock, crawler):
+        def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
             build_from_crawler(mock, crawler, *args, **kwargs)
             if hasattr(mock, "from_crawler"):
                 mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)

tests/test_utils_response.py

@@ -38,7 +38,7 @@ def test_open_in_browser():
 
     resp = Response(url, body=body)
     with pytest.raises(TypeError):
-        open_in_browser(resp, debug=True)  # pylint: disable=unexpected-keyword-arg
+        open_in_browser(resp, _openfunc=browser_open)  # type: ignore[arg-type]
 
 
 def test_get_meta_refresh():
@@ -320,4 +320,4 @@ def test_open_in_browser_text_response_uses_txt_extension():
 def test_open_in_browser_raises_for_unsupported_response_type():
     response = Response("http://www.example.com", body=b"binary")
     with pytest.raises(TypeError):
-        open_in_browser(response, _openfunc=lambda _: True)
+        open_in_browser(response, _openfunc=lambda _: True)  # type: ignore[arg-type]

tests/test_utils_signal.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import asyncio
+from typing import TYPE_CHECKING
 
 import pytest
 from pydispatch import dispatcher
@@ -16,6 +19,9 @@ from scrapy.utils.signal import (
 from scrapy.utils.test import get_from_asyncio_queue
 from tests.utils.decorators import inline_callbacks_test
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures
@@ -24,7 +30,7 @@ class TestSendCatchLog:
     @inline_callbacks_test
     def test_send_catch_log(self):
         test_signal = object()
-        handlers_called = set()
+        handlers_called: set[Callable[..., None]] = set()
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
@@ -74,7 +80,7 @@ class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
         assert arg == "test"
-        d = defer.Deferred()
+        d: defer.Deferred[str] = defer.Deferred()
         call_later(0, d.callback, "OK")
         return d
 
@@ -108,7 +114,7 @@ class TestSendCatchLogAsync2(TestSendCatchLogAsync):
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
         assert arg == "test"
-        d = defer.Deferred()
+        d: defer.Deferred[str] = defer.Deferred()
         call_later(0, d.callback, "OK")
         return d
 

tests/test_utils_spider.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from scrapy import Spider
 from scrapy.http import Request
 from scrapy.item import Item
@@ -17,7 +19,7 @@ def test_iterate_spider_output():
     r = Request("http://scrapytest.org")
     o = object()
 
-    assert list(iterate_spider_output(i)) == [i]
+    assert list(iterate_spider_output(i)) == [i]  # type: ignore[call-overload]
     assert list(iterate_spider_output(r)) == [r]
     assert list(iterate_spider_output(o)) == [o]
     assert list(iterate_spider_output([r, i, o])) == [r, i, o]

tests/test_utils_trackref.py

@@ -79,9 +79,9 @@ def test_get_oldest():
     stale entries in `live_refs`, affecting results unless explicitly cleared.
     """
 
-    def _delete_o1():
+    def _delete_o1() -> None:
         """Delete `o1` and ensure it is actually collected on PyPy."""
-        nonlocal o1
+        nonlocal o1  # type: ignore[misc]
         del o1
 
         if _IS_PYPY:
@@ -89,7 +89,7 @@ def test_get_oldest():
             # still exist until the GC runs, so we force a collection cycle.
             garbage_collect()
 
-    def _do_asserts():
+    def _do_asserts() -> None:
         assert trackref.get_oldest("Foo") is o1
         assert trackref.get_oldest("Bar") is o2
         # Ensure the newer Foo is not incorrectly considered the oldest