Disable mypy allow_untyped_calls. (#7331)

pyproject.toml

@@ -87,7 +87,6 @@ pattern = "^(?P<version>.+)$"
 [tool.mypy]
 strict = true
 allow_any_generics = true # 67 errors
-allow_untyped_calls = true  # 58 errors
 extra_checks = false # weird addErrback() errors
 untyped_calls_exclude = [
     "twisted",

scrapy/core/downloader/contextfactory.py

@@ -55,7 +55,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         *args: Any,
         **kwargs: Any,
     ):
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)  # type: ignore[no-untyped-call]
         self._ssl_method: int = method
         self.tls_verbose_logging: bool = tls_verbose_logging
         self.tls_ciphers: AcceptableCiphers

scrapy/core/downloader/handlers/http11.py

@@ -292,7 +292,7 @@ class TunnelingAgent(Agent):
         bindAddress: tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
     ):
-        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
+        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)  # type: ignore[no-untyped-call]
         self._proxyConf: tuple[str, int, bytes | None] = proxyConf
         self._contextFactory: IPolicyForHTTPS = contextFactory
 
@@ -341,7 +341,7 @@ class ScrapyProxyAgent(Agent):
         bindAddress: tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
     ):
-        super().__init__(
+        super().__init__(  # type: ignore[no-untyped-call]
             reactor=reactor,
             connectTimeout=connectTimeout,
             bindAddress=bindAddress,
@@ -363,7 +363,7 @@ class ScrapyProxyAgent(Agent):
         # connecting to a single destination, the proxy:
         return self._requestWithEndpoint(
             key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),
-            endpoint=self._getEndpoint(self._proxyURI),
+            endpoint=self._getEndpoint(self._proxyURI),  # type: ignore[no-untyped-call]
             method=method,
             parsedURI=URI.fromBytes(uri),
             headers=headers,
@@ -432,7 +432,7 @@ class ScrapyAgent:
                 pool=self._pool,
             )
 
-        return self._Agent(
+        return self._Agent(  # type: ignore[no-untyped-call]
             reactor=reactor,
             contextFactory=self._contextFactory,
             connectTimeout=timeout,

scrapy/core/downloader/tls.py

@@ -41,7 +41,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
     """
 
     def __init__(self, hostname: str, ctx: SSL.Context, verbose_logging: bool = False):
-        super().__init__(hostname, ctx)
+        super().__init__(hostname, ctx)  # type: ignore[no-untyped-call]
         self.verbose_logging: bool = verbose_logging
 
     def _identityVerifyingInfoCallback(

scrapy/core/http2/protocol.py

@@ -220,7 +220,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         assert self.transport is not None  # typing
         # Reset the idle timeout as connection is still actively sending data
-        self.resetTimeout()
+        self.resetTimeout()  # type: ignore[no-untyped-call]
 
         data = self.conn.data_to_send()
         self.transport.write(data)
@@ -247,7 +247,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         sending some data now: we should open with the connection preamble.
         """
         # Initialize the timeout
-        self.setTimeout(self.IDLE_TIMEOUT)
+        self.setTimeout(self.IDLE_TIMEOUT)  # type: ignore[no-untyped-call]
 
         assert self.transport is not None  # typing
         destination = self.transport.getPeer()
@@ -290,7 +290,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def dataReceived(self, data: bytes) -> None:
         # Reset the idle timeout as connection is still actively receiving data
-        self.resetTimeout()
+        self.resetTimeout()  # type: ignore[no-untyped-call]
 
         try:
             self._check_received_data(data)
@@ -343,7 +343,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         No need to write anything to transport here.
         """
         # Cancel the timeout if not done yet
-        self.setTimeout(None)
+        self.setTimeout(None)  # type: ignore[no-untyped-call]
 
         # Notify the connection pool instance such that no new requests are
         # sent over current connection

scrapy/utils/benchserver.py

@@ -36,7 +36,7 @@ def _getarg(
 if __name__ == "__main__":
     from twisted.internet import reactor
 
-    root = Root()
+    root = Root()  # type: ignore[no-untyped-call]
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, Site(root))
 

scrapy/utils/console.py

@@ -29,7 +29,7 @@ def _embed_ipython_shell(
 
     @wraps(_embed_ipython_shell)
     def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
-        config = load_default_config()
+        config = load_default_config()  # type: ignore[no-untyped-call]
         # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
         # and clear the instance to always have the fresh env

tests/AsyncCrawlerRunner/reactorless_datauri.py

@@ -15,7 +15,7 @@ class DataSpider(Spider):
         return {"data": response.text}
 
 
-async def main():
+async def main() -> None:
     configure_logging()
     runner = AsyncCrawlerRunner(
         settings={

tests/AsyncCrawlerRunner/reactorless_reactor.py

@@ -14,7 +14,7 @@ class NoRequestsSpider(Spider):
         yield
 
 
-async def main():
+async def main() -> None:
     configure_logging()
     runner = AsyncCrawlerRunner(
         settings={

tests/AsyncCrawlerRunner/reactorless_simple.py

@@ -15,7 +15,7 @@ class NoRequestsSpider(Spider):
         yield
 
 
-async def main():
+async def main() -> None:
     configure_logging()
     runner = AsyncCrawlerRunner(
         settings={

tests/CrawlerRunner/ip_address.py

@@ -1,5 +1,9 @@
 # ruff: noqa: E402
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.utils.reactor import install_reactor
 from tests.mockserver.dns import MockDNSServer
 from tests.mockserver.http import MockServer
@@ -17,12 +21,14 @@ from scrapy.crawler import CrawlerRunner
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import configure_logging
 
+if TYPE_CHECKING:
+    from twisted.names.common import ResolverBase
+
 
 # https://stackoverflow.com/a/32784190
-def createResolver(servers=None, resolvconf=None, hosts=None):
-    if hosts is None:
-        hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts"
-    theResolver = Resolver(resolvconf, servers)
+def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:
+    hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts"
+    theResolver = Resolver(None, servers)
     hostResolver = hostsModule.Resolver(hosts)
     chain = [hostResolver, cache.CacheResolver(), theResolver]
     return resolve.ResolverChain(chain)

tests/mockserver/http_resources.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import gzip
 import json
 import random
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
 from urllib.parse import urlencode
 
 from twisted.internet.task import deferLater
@@ -12,6 +13,16 @@ from twisted.web.util import Redirect, redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from twisted.internet.defer import Deferred
+    from twisted.web.http import Request
+
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
 
 def getarg(request, name, default=None, type_=None):
     if name in request.args:
@@ -86,7 +97,14 @@ class PayloadResource(resource.Resource):
 class LeafResource(resource.Resource):
     isLeaf = True
 
-    def deferRequest(self, request, delay, f, *a, **kw):
+    def deferRequest(
+        self,
+        request: Request,
+        delay: float,
+        f: Callable[_P, _T],
+        *a: _P.args,
+        **kw: _P.kwargs,
+    ) -> Deferred[_T]:
         from twisted.internet import reactor
 
         def _cancelrequest(_):

tests/mockserver/utils.py

@@ -1,16 +1,22 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from OpenSSL import SSL
 from twisted.internet import ssl
 
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    from twisted.internet.ssl import ContextFactory
+
 
 def ssl_context_factory(
-    keyfile="keys/localhost.key", certfile="keys/localhost.crt", cipher_string=None
-):
+    keyfile: str = "keys/localhost.key",
+    certfile: str = "keys/localhost.crt",
+    cipher_string: str | None = None,
+) -> ContextFactory:
     factory = ssl.DefaultOpenSSLContextFactory(
         str(Path(__file__).parent.parent / keyfile),
         str(Path(__file__).parent.parent / certfile),

tests/spiders.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import asyncio
 import time
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode
 
 from twisted.internet import defer
@@ -20,9 +21,18 @@ from scrapy.spiders.crawl import CrawlSpider, Rule
 from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
 from scrapy.utils.test import get_from_asyncio_queue
 
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
 
 class MockServerSpider(Spider):
-    def __init__(self, *args, mockserver=None, is_secure=False, **kwargs):
+    def __init__(
+        self,
+        *args,
+        mockserver: MockServer | None = None,
+        is_secure: bool = False,
+        **kwargs: Any,
+    ):
         super().__init__(*args, **kwargs)
         self.mockserver = mockserver
         self.is_secure = is_secure
@@ -31,9 +41,9 @@ class MockServerSpider(Spider):
 class MetaSpider(MockServerSpider):
     name = "meta"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
-        self.meta = {}
+        self.meta: dict[str, Any] = {}
 
     def closed(self, reason):
         self.meta["close_reason"] = reason

tests/test_command_shell.py

@@ -145,6 +145,6 @@ class TestInteractiveShell:
         p.sendline("type(response)")
         p.expect_exact("HtmlResponse")
         p.sendeof()
-        p.wait()
+        p.wait()  # type: ignore[no-untyped-call]
         logfile.seek(0)
         assert "Traceback" not in logfile.read().decode()

tests/test_core_downloader.py

@@ -29,6 +29,7 @@ from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
+    from twisted.internet.ssl import ContextFactory
     from twisted.web.iweb import IBodyProducer
 
 
@@ -40,7 +41,7 @@ class TestSlot:
 
 @pytest.mark.requires_reactor
 class TestContextFactoryBase:
-    context_factory = None
+    context_factory: ContextFactory | None = None
 
     @async_yield_fixture
     async def server_url(self, tmp_path):

tests/test_crawl.py

@@ -304,7 +304,8 @@ with multiples lines
             )
         self._assert_retried(log)
 
-    def _assert_retried(self, log):
+    @staticmethod
+    def _assert_retried(log: LogCapture | str) -> None:
         assert str(log).count("Retrying") == 2
         assert str(log).count("Gave up retrying") == 1
 

tests/test_crawler.py

@@ -4,7 +4,7 @@ import re
 import warnings
 from collections.abc import Generator
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import pytest
 from twisted.internet.defer import Deferred
@@ -18,6 +18,7 @@ from scrapy.crawler import (
     Crawler,
     CrawlerProcess,
     CrawlerRunner,
+    CrawlerRunnerBase,
 )
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.throttle import AutoThrottle
@@ -45,7 +46,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None):
 
 
 class TestBaseCrawler:
-    def assertOptionIsDefault(self, settings, key):
+    def assertOptionIsDefault(self, settings: Settings, key: str) -> None:
         assert isinstance(settings, Settings)
         assert settings[key] == getattr(default_settings, key)
 
@@ -668,12 +669,12 @@ class NoRequestsSpider(scrapy.Spider):
 @pytest.mark.requires_reactor
 class TestCrawlerRunnerHasSpider:
     @staticmethod
-    def _runner():
+    def _runner() -> CrawlerRunnerBase:
         return CrawlerRunner(get_reactor_settings())
 
     @staticmethod
-    def _crawl(runner, spider):
-        return runner.crawl(spider)
+    def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
+        return cast("Deferred[None]", runner.crawl(spider))
 
     @inline_callbacks_test
     def test_crawler_runner_bootstrap_successful(self):
@@ -742,11 +743,11 @@ class TestCrawlerRunnerHasSpider:
 @pytest.mark.only_asyncio
 class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
     @staticmethod
-    def _runner():
+    def _runner() -> CrawlerRunnerBase:
         return AsyncCrawlerRunner(get_reactor_settings())
 
     @staticmethod
-    def _crawl(runner, spider):
+    def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
         return deferred_from_coro(runner.crawl(spider))
 
     def test_crawler_runner_asyncio_enabled_true(self):

tests/test_downloaderslotssettings.py

@@ -33,6 +33,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
 
     def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
+        assert self.mockserver
         self.default_slot = self.mockserver.host
         self.times: dict[str, list[float]] = {}
 

tests/test_engine.py

@@ -7,7 +7,7 @@ import sys
 from collections import defaultdict
 from dataclasses import dataclass
 from logging import DEBUG
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Any, cast
 from unittest.mock import Mock, call
 from urllib.parse import urlparse
 
@@ -22,7 +22,7 @@ from scrapy import signals
 from scrapy.core.engine import ExecutionEngine, _Slot
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.exceptions import CloseSpider, IgnoreRequest
-from scrapy.http import Request, Response
+from scrapy.http import Headers, Request, Response
 from scrapy.item import Field, Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
@@ -38,6 +38,8 @@ from tests import get_testdata
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
     from scrapy.core.scheduler import Scheduler
     from scrapy.crawler import Crawler
     from scrapy.statscollectors import MemoryStatsCollector
@@ -133,16 +135,16 @@ class ChangeCloseReasonSpider(MySpider):
 class CrawlerRun:
     """A class to run the crawler and keep track of events occurred"""
 
-    def __init__(self, spider_class):
-        self.respplug = []
-        self.reqplug = []
-        self.reqdropped = []
-        self.reqreached = []
-        self.itemerror = []
-        self.itemresp = []
-        self.headers = {}
-        self.bytes = defaultdict(list)
-        self.signals_caught = {}
+    def __init__(self, spider_class: type[Spider]):
+        self.respplug: list[tuple[Response, Spider]] = []
+        self.reqplug: list[tuple[Request, Spider]] = []
+        self.reqdropped: list[tuple[Request, Spider]] = []
+        self.reqreached: list[tuple[Request, Spider]] = []
+        self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
+        self.itemresp: list[tuple[Any, Response]] = []
+        self.headers: dict[Request, Headers] = {}
+        self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
+        self.signals_caught: dict[Any, dict[str, Any]] = {}
         self.spider_class = spider_class
 
     async def run(self, mockserver: MockServer) -> None:
@@ -188,35 +190,39 @@ class CrawlerRun:
     def geturl(self, path: str) -> str:
         return self.mockserver.url(path)
 
-    def getpath(self, url):
+    def getpath(self, url: str) -> str:
         u = urlparse(url)
         return u.path
 
-    def item_error(self, item, response, spider, failure):
+    def item_error(
+        self, item: Any, response: Response, spider: Spider, failure: Failure
+    ) -> None:
         self.itemerror.append((item, response, spider, failure))
 
-    def item_scraped(self, item, spider, response):
+    def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
         self.itemresp.append((item, response))
 
-    def headers_received(self, headers, body_length, request, spider):
+    def headers_received(
+        self, headers: Headers, body_length: int, request: Request, spider: Spider
+    ) -> None:
         self.headers[request] = headers
 
-    def bytes_received(self, data, request, spider):
+    def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
         self.bytes[request].append(data)
 
-    def request_scheduled(self, request, spider):
+    def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.reqplug.append((request, spider))
 
-    def request_reached(self, request, spider):
+    def request_reached(self, request: Request, spider: Spider) -> None:
         self.reqreached.append((request, spider))
 
-    def request_dropped(self, request, spider):
+    def request_dropped(self, request: Request, spider: Spider) -> None:
         self.reqdropped.append((request, spider))
 
-    def response_downloaded(self, response, spider):
+    def response_downloaded(self, response: Response, spider: Spider) -> None:
         self.respplug.append((response, spider))
 
-    def record_signal(self, *args, **kwargs):
+    def record_signal(self, *args: Any, **kwargs: Any) -> None:
         """Record a signal and its parameters"""
         signalargs = kwargs.copy()
         sig = signalargs.pop("signal")
@@ -294,8 +300,8 @@ class TestEngineBase:
     @staticmethod
     def _assert_scraped_items(run: CrawlerRun) -> None:
         assert len(run.itemresp) == 2
-        for item, response in run.itemresp:
-            item = ItemAdapter(item)
+        for item_, response in run.itemresp:
+            item = ItemAdapter(item_)
             assert item["url"] == response.url
             if "item1.html" in item["url"]:
                 assert item["name"] == "Item 1 name"
@@ -308,6 +314,7 @@ class TestEngineBase:
     def _assert_headers_received(run: CrawlerRun) -> None:
         for headers in run.headers.values():
             assert b"Server" in headers
+            assert headers[b"Server"]
             assert b"TwistedWeb" in headers[b"Server"]
             assert b"Date" in headers
             assert b"Content-Type" in headers

tests/test_extension_periodic_log.py

@@ -59,10 +59,10 @@ stats_dump_2 = {
 
 
 class CustomPeriodicLog(PeriodicLog):
-    def set_a(self):
+    def set_a(self) -> None:
         self.stats._stats = stats_dump_1
 
-    def set_b(self):
+    def set_b(self) -> None:
         self.stats._stats = stats_dump_2
 
 
@@ -90,7 +90,9 @@ class TestPeriodicLog:
 
     @pytest.mark.requires_reactor  # needs a reactor or an event loop for PeriodicLog.task
     def test_log_delta(self):
-        def emulate(settings=None):
+        def emulate(
+            settings: dict[str, Any] | None = None,
+        ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
             spider = MetaSpider()
             ext = extension(settings)
             ext.spider_opened(spider)
@@ -154,7 +156,9 @@ class TestPeriodicLog:
 
     @pytest.mark.requires_reactor  # needs a reactor or an event loop for PeriodicLog.task
     def test_log_stats(self):
-        def emulate(settings=None):
+        def emulate(
+            settings: dict[str, Any] | None = None,
+        ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
             spider = MetaSpider()
             ext = extension(settings)
             ext.spider_opened(spider)

tests/test_feedexport.py

@@ -11,7 +11,7 @@ from abc import ABC, abstractmethod
 from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
-from typing import TYPE_CHECKING, Any
+from typing import IO, TYPE_CHECKING, Any
 from unittest import mock
 from urllib.parse import urljoin
 from urllib.request import pathname2url
@@ -44,12 +44,12 @@ if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
 
 
-def path_to_url(path):
+def path_to_url(path: Path) -> str:
     return urljoin("file:", pathname2url(str(path)))
 
 
-def printf_escape(string):
-    return string.replace("%", "%%")
+def printf_escape(s: str) -> str:
+    return s.replace("%", "%%")
 
 
 class FromCrawlerMixin:
@@ -239,8 +239,10 @@ class TestFeedExportBase(ABC):
     ) -> dict[str, Any]:
         pass
 
-    def _load_until_eof(self, data, load_func):
-        result = []
+    def _load_until_eof(
+        self, data: bytes, load_func: Callable[[IO[bytes]], Any]
+    ) -> list[Any]:
+        result: list[Any] = []
         with tempfile.TemporaryFile() as temp:
             temp.write(data)
             temp.seek(0)

tests/test_http2_client_protocol.py

@@ -411,9 +411,22 @@ class TestHttps2ClientProtocol:
             client, request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200
         )
 
-    async def _check_POST_json_x10(self, *args, **kwargs):
+    async def _check_POST_json_x10(
+        self,
+        client: H2ClientProtocol,
+        request: Request,
+        expected_request_body: dict[str, str],
+        expected_extra_data: str,
+        expected_status: int,
+    ) -> None:
         async def get_coro() -> None:
-            await self._check_POST_json(*args, **kwargs)
+            await self._check_POST_json(
+                client,
+                request,
+                expected_request_body,
+                expected_extra_data,
+                expected_status,
+            )
 
         await self._check_repeat(get_coro, 10)
 
@@ -717,7 +730,7 @@ class TestHttps2ClientProtocol:
         request = Request(self.get_url(server_port, "/timeout"))
 
         # Update the timer to 1s to test connection timeout
-        client.setTimeout(1)
+        client.setTimeout(1)  # type: ignore[no-untyped-call]
 
         with pytest.raises(ResponseFailed) as exc_info:
             yield make_request_dfd(client, request)

tests/test_pipelines.py

@@ -253,7 +253,7 @@ class TestCustomPipelineManager:
     @pytest.mark.requires_reactor
     def test_deprecated_process_item_spider_arg(self) -> None:
         class CustomPipelineManager(ItemPipelineManager):
-            def process_item(self, item, spider):  # pylint: disable=useless-parent-delegation
+            def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:  # pylint: disable=useless-parent-delegation
                 return super().process_item(item, spider)
 
         crawler = get_crawler(DefaultSpider)

tests/test_robotstxt_interface.py

@@ -8,7 +8,7 @@ from scrapy.robotstxt import (
 )
 
 
-def rerp_available():
+def rerp_available() -> bool:
     # check if robotexclusionrulesparser is installed
     try:
         from robotexclusionrulesparser import (  # noqa: PLC0415