Merge pull request #5334 from wRAR/mw-manager-typing-fix

Fix typing of middleware methods.

scrapy/core/downloader/middleware.py

@@ -3,7 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-from typing import Callable, Union
+from typing import Callable, Union, cast
 
 from twisted.internet import defer
 from twisted.python.failure import Failure
@@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         @defer.inlineCallbacks
         def process_request(request: Request):
             for method in self.methods['process_request']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
@@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 return response
 
             for method in self.methods['process_response']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
@@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         def process_exception(failure: Failure):
             exception = failure.value
             for method in self.methods['process_exception']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(

scrapy/core/spidermw.py

@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union
+from typing import Any, Callable, Generator, Iterable, Union, cast
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
@@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                               spider: Spider) -> Any:
         for method in self.methods['process_spider_input']:
+            method = cast(Callable, method)
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:

scrapy/middleware.py

@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict
+from typing import Callable, Deque, Dict, Optional, cast, Iterable
 
 from twisted.internet.defer import Deferred
 
@@ -9,7 +9,7 @@ from scrapy import Spider
 from scrapy.exceptions import NotConfigured
 from scrapy.settings import Settings
 from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
+from scrapy.utils.defer import process_parallel, process_chain
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +21,8 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
+        # Optional because process_spider_output and process_spider_exception can be None
+        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -64,14 +65,12 @@ class MiddlewareManager:
             self.methods['close_spider'].appendleft(mw.close_spider)
 
     def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
-        return process_parallel(self.methods[methodname], obj, *args)
+        methods = cast(Iterable[Callable], self.methods[methodname])
+        return process_parallel(methods, obj, *args)
 
     def _process_chain(self, methodname: str, obj, *args) -> Deferred:
-        return process_chain(self.methods[methodname], obj, *args)
-
-    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
-        return process_chain_both(self.methods[cb_methodname],
-                                  self.methods[eb_methodname], obj, *args)
+        methods = cast(Iterable[Callable], self.methods[methodname])
+        return process_chain(methods, obj, *args)
 
     def open_spider(self, spider: Spider) -> Deferred:
         return self._process_parallel('open_spider', spider)