Update type hints for Twisted 23.8.0. (#6034)

scrapy/core/downloader/__init__.py

@@ -154,7 +154,7 @@ class Downloader:
         self.signals.send_catch_log(
             signal=signals.request_reached_downloader, request=request, spider=spider
         )
-        deferred = Deferred().addBoth(_deactivate)
+        deferred: Deferred = Deferred().addBoth(_deactivate)
         slot.queue.append((request, deferred))
         self._process_queue(spider, slot)
         return deferred

scrapy/core/engine.py

@@ -220,7 +220,12 @@ class ExecutionEngine:
                 extra={"spider": self.spider},
             )
         )
-        d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))  # type: ignore[arg-type]
+
+        def _remove_request(_: Any) -> None:
+            assert self.slot
+            self.slot.remove_request(request)
+
+        d.addBoth(_remove_request)
         d.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -300,8 +305,8 @@ class ExecutionEngine:
         return self._download(request).addBoth(self._downloaded, request)
 
     def _downloaded(
-        self, result: Union[Response, Request], request: Request
-    ) -> Union[Deferred, Response]:
+        self, result: Union[Response, Request, Failure], request: Request
+    ) -> Union[Deferred, Response, Failure]:
         assert self.slot is not None  # typing
         self.slot.remove_request(request)
         return self.download(result) if isinstance(result, Request) else result

scrapy/core/scraper.py

@@ -221,7 +221,11 @@ class Scraper:
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(
-        self, _failure: Failure, request: Request, response: Response, spider: Spider
+        self,
+        _failure: Failure,
+        request: Request,
+        response: Union[Response, Failure],
+        spider: Spider,
     ) -> None:
         exc = _failure.value
         if isinstance(exc, CloseSpider):
@@ -248,7 +252,7 @@ class Scraper:
         self,
         result: Union[Iterable, AsyncIterable],
         request: Request,
-        response: Response,
+        response: Union[Response, Failure],
         spider: Spider,
     ) -> Deferred:
         if not result:

scrapy/logformatter.py

@@ -130,7 +130,11 @@ class LogFormatter:
         }
 
     def spider_error(
-        self, failure: Failure, request: Request, response: Response, spider: Spider
+        self,
+        failure: Failure,
+        request: Request,
+        response: Union[Response, Failure],
+        spider: Spider,
     ) -> dict:
         """Logs an error message from a spider.