From 4cc039628eb4861f98f7997e90125745e30f8687 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 19:52:03 +0500 Subject: [PATCH 1/2] Fix typing of middleware methods. --- scrapy/core/downloader/middleware.py | 5 ++++- scrapy/core/spidermw.py | 3 ++- scrapy/middleware.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..e8f60287a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,17 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) + eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) + return process_chain_both(cb_methods, eb_methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From a2763c608d33a9254034d650457b7efa7b434ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 18:35:00 +0500 Subject: [PATCH 2/2] Remove unused MiddlewareManager._process_chain_both(). --- scrapy/middleware.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index e8f60287a..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -72,11 +72,6 @@ class MiddlewareManager: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) - eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) - return process_chain_both(cb_methods, eb_methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider)