Fix most of the new typing issues in utils.

scrapy/utils/defer.py

@@ -10,6 +10,7 @@ from typing import (
     AsyncGenerator,
     AsyncIterable,
     Callable,
+    cast,
     Coroutine,
     Generator,
     Iterable,
@@ -38,8 +39,8 @@ def defer_fail(_failure: Failure) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
-    reactor.callLater(0.1, d.errback, _failure)
+    d: Deferred = Deferred()
+    reactor.callLater(0.1, d.errback, _failure)  # type: ignore[attr-defined]
     return d
 
 
@@ -52,8 +53,8 @@ def defer_succeed(result) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
-    reactor.callLater(0.1, d.callback, result)
+    d: Deferred = Deferred()
+    reactor.callLater(0.1, d.callback, result)  # type: ignore[attr-defined]
     return d
 
 
@@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = cast(
+            Deferred, deferred_from_coro(self.aiterator.__anext__())
+        )
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d = Deferred()
+        d: Deferred = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
@@ -209,7 +212,7 @@ def parallel_async(
 
 def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for x in callbacks:
         d.addCallback(x, *a, **kw)
     d.callback(input)
@@ -220,7 +223,7 @@ def process_chain_both(
     callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallbacks(
             callback=cb,

scrapy/utils/response.py

@@ -6,7 +6,7 @@ import os
 import re
 import tempfile
 import webbrowser
-from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, Tuple, Union
 from weakref import WeakKeyDictionary
 
 from twisted.web import http
@@ -38,12 +38,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa
 
 def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
-    ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
+    ignore_tags: Iterable[str] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        _metaref_cache[response] = html.get_meta_refresh(
+        # a w3lib typing bug here
+        _metaref_cache[response] = html.get_meta_refresh(  # type: ignore[assignment]
             text, response.url, response.encoding, ignore_tags=ignore_tags
         )
     return _metaref_cache[response]

scrapy/utils/testsite.py

@@ -50,6 +50,6 @@ def test_site():
 if __name__ == "__main__":
     from twisted.internet import reactor
 
-    port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
+    port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")  # type: ignore[attr-defined]
     print(f"http://localhost:{port.getHost().port}/")
-    reactor.run()
+    reactor.run()  # type: ignore[attr-defined]