From 2ad81a0ef88309526b52a03d5281600e84ef94a5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Sep 2025 20:51:47 +0400 Subject: [PATCH] Deprecate the spider arg of downloader middleware methods. (#7037) * Deprecate the spider arg of downloader middleware methods. * Add a test for deprecated spider args in downloader mws. --- docs/topics/downloader-middleware.rst | 15 +- scrapy/core/downloader/middleware.py | 36 ++- scrapy/downloadermiddlewares/cookies.py | 25 +- .../downloadermiddlewares/defaultheaders.py | 4 +- .../downloadermiddlewares/downloadtimeout.py | 4 +- scrapy/downloadermiddlewares/httpauth.py | 4 +- scrapy/downloadermiddlewares/httpcache.py | 29 +- .../downloadermiddlewares/httpcompression.py | 7 +- scrapy/downloadermiddlewares/httpproxy.py | 4 +- scrapy/downloadermiddlewares/offsite.py | 22 +- scrapy/downloadermiddlewares/redirect.py | 29 +- scrapy/downloadermiddlewares/retry.py | 25 +- scrapy/downloadermiddlewares/robotstxt.py | 37 ++- scrapy/downloadermiddlewares/stats.py | 10 +- scrapy/downloadermiddlewares/useragent.py | 4 +- tests/test_downloadermiddleware.py | 58 +++- tests/test_downloadermiddleware_cookies.py | 158 +++++----- ...est_downloadermiddleware_defaultheaders.py | 13 +- ...st_downloadermiddleware_downloadtimeout.py | 8 +- tests/test_downloadermiddleware_httpauth.py | 20 +- tests/test_downloadermiddleware_httpcache.py | 184 +++++++----- ...st_downloadermiddleware_httpcompression.py | 61 ++-- tests/test_downloadermiddleware_httpproxy.py | 100 +++---- tests/test_downloadermiddleware_offsite.py | 68 ++--- tests/test_downloadermiddleware_redirect.py | 283 ++++++++---------- tests/test_downloadermiddleware_retry.py | 65 ++-- tests/test_downloadermiddleware_robotstxt.py | 43 ++- tests/test_downloadermiddleware_stats.py | 7 +- tests/test_downloadermiddleware_useragent.py | 12 +- tests/test_middleware.py | 4 +- tests/test_request_attribute_binding.py | 19 +- tests/test_request_cb_kwargs.py | 4 +- tests/test_spidermiddleware_referer.py | 31 +- 33 files changed, 728 insertions(+), 665 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 60b6aab78..6f36acc68 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -70,7 +70,7 @@ defines one or more of these methods: .. note:: Any of the downloader middleware methods may also return a deferred. - .. method:: process_request(request, spider) + .. method:: process_request(request) This method is called for each request that goes through the download middleware. @@ -102,10 +102,7 @@ defines one or more of these methods: :param request: the request being processed :type request: :class:`~scrapy.Request` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_response(request, response, spider) + .. method:: process_response(request, response) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object or @@ -129,10 +126,7 @@ defines one or more of these methods: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_exception(request, exception, spider) + .. method:: process_exception(request, exception) Scrapy calls :meth:`process_exception` when a download handler or a :meth:`process_request` (from a downloader middleware) raises an @@ -160,9 +154,6 @@ defines one or more of these methods: :param exception: the raised exception :type exception: an ``Exception`` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 47175fb13..a822ba811 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -35,10 +35,13 @@ class DownloaderMiddlewareManager(MiddlewareManager): def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_request"): self.methods["process_request"].append(mw.process_request) + self._check_mw_method_spider_arg(mw.process_request) if hasattr(mw, "process_response"): self.methods["process_response"].appendleft(mw.process_response) + self._check_mw_method_spider_arg(mw.process_response) if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) + self._check_mw_method_spider_arg(mw.process_exception) @inlineCallbacks def download( @@ -64,9 +67,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_request"]: method = cast("Callable", method) - response = yield deferred_from_coro( - method(request=request, spider=self._spider) - ) + if method in self._mw_methods_requiring_spider: + response = yield deferred_from_coro( + method(request=request, spider=self._spider) + ) + else: + response = yield deferred_from_coro(method(request=request)) if response is not None and not isinstance( response, (Response, Request) ): @@ -91,9 +97,14 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods["process_response"]: method = cast("Callable", method) - response = yield deferred_from_coro( - method(request=request, response=response, spider=self._spider) - ) + if method in self._mw_methods_requiring_spider: + response = yield deferred_from_coro( + method(request=request, response=response, spider=self._spider) + ) + else: + response = yield deferred_from_coro( + method(request=request, response=response) + ) if not isinstance(response, (Response, Request)): raise _InvalidOutput( f"Middleware {method.__qualname__} must return Response or Request, " @@ -109,9 +120,16 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_exception"]: method = cast("Callable", method) - response = yield deferred_from_coro( - method(request=request, exception=exception, spider=self._spider) - ) + if method in self._mw_methods_requiring_spider: + response = yield deferred_from_coro( + method( + request=request, exception=exception, spider=self._spider + ) + ) + else: + response = yield deferred_from_coro( + method(request=request, exception=exception) + ) if response is not None and not isinstance( response, (Response, Request) ): diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 9156b8c3a..d945546d5 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -9,6 +9,7 @@ from tldextract import TLDExtract from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -39,6 +40,8 @@ def _is_public_domain(domain: str) -> bool: class CookiesMiddleware: """This middleware enables working with sites that need cookies""" + crawler: Crawler + def __init__(self, debug: bool = False): self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug @@ -47,7 +50,9 @@ class CookiesMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured - return cls(crawler.settings.getbool("COOKIES_DEBUG")) + o = cls(crawler.settings.getbool("COOKIES_DEBUG")) + o.crawler = crawler + return o def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request @@ -67,8 +72,9 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: if request.meta.get("dont_merge_cookies", False): return None @@ -81,11 +87,12 @@ class CookiesMiddleware: # set Cookie header request.headers.pop("Cookie", None) jar.add_cookie_header(request) - self._debug_cookie(request, spider) + self._debug_cookie(request) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if request.meta.get("dont_merge_cookies", False): return response @@ -96,11 +103,11 @@ class CookiesMiddleware: cookies = jar.make_cookies(response, request) self._process_cookies(cookies, jar=jar, request=request) - self._debug_set_cookie(response, spider) + self._debug_set_cookie(response) return response - def _debug_cookie(self, request: Request, spider: Spider) -> None: + def _debug_cookie(self, request: Request) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -109,9 +116,9 @@ class CookiesMiddleware: if cl: cookies = "\n".join(f"Cookie: {c}\n" for c in cl) msg = f"Sending cookies to: {request}\n{cookies}" - logger.debug(msg, extra={"spider": spider}) + logger.debug(msg, extra={"spider": self.crawler.spider}) - def _debug_set_cookie(self, response: Response, spider: Spider) -> None: + def _debug_set_cookie(self, response: Response) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -120,7 +127,7 @@ class CookiesMiddleware: if cl: cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) msg = f"Received cookies from: {response}\n{cookies}" - logger.debug(msg, extra={"spider": spider}) + logger.debug(msg, extra={"spider": self.crawler.spider}) def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None: """ diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index d58b4490b..7c940a078 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,6 +8,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.python import without_none_values if TYPE_CHECKING: @@ -30,8 +31,9 @@ class DefaultHeadersMiddleware: headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: for k, v in self._headers: request.headers.setdefault(k, v) diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 28456c697..b57d5c2a9 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -9,6 +9,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy import Request, Spider, signals +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -31,8 +32,9 @@ class DownloadTimeoutMiddleware: def spider_opened(self, spider: Spider) -> None: self._timeout = getattr(spider, "download_timeout", self._timeout) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 80107261b..c28c93d4e 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: @@ -38,8 +39,9 @@ class HttpAuthMiddleware: self.auth = basic_auth_header(usr, pwd) self.domain = spider.http_auth_domain # type: ignore[attr-defined] + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: auth = getattr(self, "auth", None) if ( diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index d3bfe6eba..453e8a6a3 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -17,6 +17,7 @@ from twisted.web.client import ResponseFailed from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object if TYPE_CHECKING: @@ -45,6 +46,8 @@ class HttpCacheMiddleware: OSError, ) + crawler: Crawler + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool("HTTPCACHE_ENABLED"): raise NotConfigured @@ -59,6 +62,7 @@ class HttpCacheMiddleware: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + o.crawler = crawler return o def spider_opened(self, spider: Spider) -> None: @@ -67,8 +71,9 @@ class HttpCacheMiddleware: def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: if request.meta.get("dont_cache", False): return None @@ -80,7 +85,7 @@ class HttpCacheMiddleware: # Look for cached response and check if expired cachedresponse: Response | None = self.storage.retrieve_response( - spider, request + self.crawler.spider, request ) if cachedresponse is None: self.stats.inc_value("httpcache/miss") @@ -101,8 +106,9 @@ class HttpCacheMiddleware: return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if request.meta.get("dont_cache", False): return response @@ -121,7 +127,7 @@ class HttpCacheMiddleware: cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand") - self._cache_response(spider, response, request, cachedresponse) + self._cache_response(response, request) return response if self.policy.is_cached_response_valid(cachedresponse, response, request): @@ -129,11 +135,12 @@ class HttpCacheMiddleware: return cachedresponse self.stats.inc_value("httpcache/invalidate") - self._cache_response(spider, response, request, cachedresponse) + self._cache_response(response, request) return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider + self, request: Request, exception: Exception, spider: Spider | None = None ) -> Request | Response | None: cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( @@ -143,15 +150,9 @@ class HttpCacheMiddleware: return cachedresponse return None - def _cache_response( - self, - spider: Spider, - response: Response, - request: Request, - cachedresponse: Response | None, - ) -> None: + def _cache_response(self, response: Response, request: Request) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value("httpcache/store") - self.storage.store_response(spider, request, response) + self.storage.store_response(self.crawler.spider, request, response) else: self.stats.inc_value("httpcache/uncacheable") diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index e3326146e..f729649a0 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -15,6 +15,7 @@ from scrapy.utils._compression import ( _unbrotli, _unzstd, ) +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.gz import gunzip if TYPE_CHECKING: @@ -93,14 +94,16 @@ class HttpCompressionMiddleware: ) self._warn_size = spider.download_warnsize + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if request.method == "HEAD": return response diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index cb7fa8c90..d3d46a947 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -10,6 +10,7 @@ from urllib.request import ( # type: ignore[attr-defined] ) from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -55,8 +56,9 @@ class HttpProxyMiddleware: return creds, proxy_url + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 682f2cb06..33d7ba609 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: @@ -21,29 +22,34 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: + crawler: Crawler + + def __init__(self, stats: StatsCollector): + self.stats = stats + self.domains_seen: set[str] = set() + @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) + o.crawler = crawler return o - def __init__(self, stats: StatsCollector): - self.stats = stats - self.domains_seen: set[str] = set() - def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) def request_scheduled(self, request: Request, spider: Spider) -> None: - self.process_request(request, spider) + self.process_request(request) - def process_request(self, request: Request, spider: Spider) -> None: + @_warn_spider_arg + def process_request(self, request: Request, spider: Spider | None = None) -> None: + assert self.crawler.spider if ( request.dont_filter or request.meta.get("allow_offsite") - or self.should_follow(request, spider) + or self.should_follow(request, self.crawler.spider) ): return domain = urlparse_cached(request).hostname @@ -52,7 +58,7 @@ class OffsiteMiddleware: logger.debug( "Filtered offsite request to %(domain)r: %(request)s", {"domain": domain, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) self.stats.inc_value("offsite/domains") self.stats.inc_value("offsite/filtered") diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 530cccb53..61f62ecfc 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -8,6 +8,7 @@ from w3lib.url import safe_url_string from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh @@ -79,6 +80,7 @@ def _build_redirect_request( class BaseRedirectMiddleware: + crawler: Crawler enabled_setting: str = "REDIRECT_ENABLED" def __init__(self, settings: BaseSettings): @@ -90,11 +92,11 @@ class BaseRedirectMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + o = cls(crawler.settings) + o.crawler = crawler + return o - def _redirect( - self, redirected: Request, request: Request, spider: Spider, reason: Any - ) -> Request: + def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 @@ -114,13 +116,13 @@ class BaseRedirectMiddleware: logger.debug( "Redirecting (%(reason)s) to %(redirected)s from %(request)s", {"reason": reason, "redirected": redirected, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) return redirected logger.debug( "Discarding %(request)s: max redirections reached", {"request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) raise IgnoreRequest("max redirections reached") @@ -144,12 +146,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): and meta-refresh html tag. """ + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if ( request.meta.get("dont_redirect", False) - or response.status in getattr(spider, "handle_httpstatus_list", []) + or response.status + in getattr(self.crawler.spider, "handle_httpstatus_list", []) or response.status in request.meta.get("handle_httpstatus_list", []) or request.meta.get("handle_httpstatus_all", False) ): @@ -171,10 +175,10 @@ class RedirectMiddleware(BaseRedirectMiddleware): return response if response.status in (301, 307, 308) or request.method == "HEAD": - return self._redirect(redirected, request, spider, response.status) + return self._redirect(redirected, request, response.status) redirected = self._redirect_request_using_get(request, redirected_url) - return self._redirect(redirected, request, spider, response.status) + return self._redirect(redirected, request, response.status) class MetaRefreshMiddleware(BaseRedirectMiddleware): @@ -185,8 +189,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if ( request.meta.get("dont_redirect", False) @@ -203,5 +208,5 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): if urlparse_cached(redirected).scheme not in {"http", "https"}: return response if cast("float", interval) < self._maxdelay: - return self._redirect(redirected, request, spider, "meta refresh") + return self._redirect(redirected, request, "meta refresh") return response diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 723fe5e93..b6dba5773 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -16,6 +16,7 @@ from logging import Logger, getLogger from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -123,6 +124,8 @@ def get_retry_request( class RetryMiddleware: + crawler: Crawler + def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured @@ -136,39 +139,41 @@ class RetryMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + o = cls(crawler.settings) + o.crawler = crawler + return o + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: reason = response_status_message(response.status) - return self._retry(request, reason, spider) or response + return self._retry(request, reason) or response return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider + self, request: Request, exception: Exception, spider: Spider | None = None ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): - return self._retry(request, exception, spider) + return self._retry(request, exception) return None def _retry( - self, - request: Request, - reason: str | Exception | type[Exception], - spider: Spider, + self, request: Request, reason: str | Exception | type[Exception] ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) + assert self.crawler.spider return get_retry_request( request, reason=reason, - spider=spider, + spider=self.crawler.spider, max_retry_times=max_retry_times, priority_adjust=priority_adjust, ) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 83af0f7bf..7d0c17884 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -14,6 +14,7 @@ from twisted.internet.defer import Deferred from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -51,17 +52,18 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - async def process_request(self, request: Request, spider: Spider) -> None: + @_warn_spider_arg + async def process_request( + self, request: Request, spider: Spider | None = None + ) -> None: if request.meta.get("dont_obey_robotstxt"): return if request.url.startswith("data:") or request.url.startswith("file:"): return - rp = await self.robot_parser(request, spider) - self.process_request_2(rp, request, spider) + rp = await self.robot_parser(request) + self.process_request_2(rp, request) - def process_request_2( - self, rp: RobotParser | None, request: Request, spider: Spider - ) -> None: + def process_request_2(self, rp: RobotParser | None, request: Request) -> None: if rp is None: return @@ -73,15 +75,13 @@ class RobotsTxtMiddleware: logger.debug( "Forbidden by robots.txt: %(request)s", {"request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") - async def robot_parser( - self, request: Request, spider: Spider - ) -> RobotParser | None: + async def robot_parser(self, request: Request) -> RobotParser | None: url = urlparse_cached(request) netloc = url.netloc @@ -100,7 +100,13 @@ class RobotsTxtMiddleware: resp = await self.crawler.engine.download_async(robotsreq) self._parse_robots(resp, netloc) except Exception as e: - self._logerror(e, robotsreq, spider) + if not isinstance(e, IgnoreRequest): + logger.error( + "Error downloading %(request)s: %(f_exception)s", + {"request": request, "f_exception": e}, + exc_info=True, + extra={"spider": self.crawler.spider}, + ) self._robots_error(e, netloc) self.crawler.stats.inc_value("robotstxt/request_count") @@ -109,15 +115,6 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _logerror(self, exc: Exception, request: Request, spider: Spider) -> None: - if not isinstance(exc, IgnoreRequest): - logger.error( - "Error downloading %(request)s: %(f_exception)s", - {"request": request, "f_exception": exc}, - exc_info=True, # noqa: LOG014 - extra={"spider": spider}, - ) - def _parse_robots(self, response: Response, netloc: str) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 38e9ad891..bafa931de 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from twisted.web import http from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -45,8 +46,9 @@ class DownloaderStats: assert crawler.stats return cls(crawler.stats) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: self.stats.inc_value("downloader/request_count") self.stats.inc_value(f"downloader/request_method_count/{request.method}") @@ -54,8 +56,9 @@ class DownloaderStats: self.stats.inc_value("downloader/request_bytes", reqlen) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider + self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: self.stats.inc_value("downloader/response_count") self.stats.inc_value(f"downloader/response_status_count/{response.status}") @@ -69,8 +72,9 @@ class DownloaderStats: self.stats.inc_value("downloader/response_bytes", reslen) return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider + self, request: Request, exception: Exception, spider: Spider | None = None ) -> Request | Response | None: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count") diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index ba379f862..c43a0195c 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy import Request, Spider, signals +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -29,8 +30,9 @@ class UserAgentMiddleware: def spider_opened(self, spider: Spider) -> None: self.user_agent = getattr(spider, "user_agent", self.user_agent) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider + self, request: Request, spider: Spider | None = None ) -> Request | Response | None: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 2684cc1de..f50e7bec7 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -128,7 +128,7 @@ class TestResponseFromProcessRequest(TestManagerBase): download_func = mock.MagicMock() class ResponseMiddleware: - def process_request(self, request, spider): + def process_request(self, request): return resp async with self.get_mwman() as mwman: @@ -151,11 +151,11 @@ class TestResponseFromProcessException(TestManagerBase): raise ValueError("test") class ResponseMiddleware: - def process_response(self, request, response, spider): + def process_response(self, request, response): calls.append("process_response") return resp - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): calls.append("process_exception") return resp @@ -176,7 +176,7 @@ class TestInvalidOutput(TestManagerBase): req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: - def process_request(self, request, spider): + def process_request(self, request): return 1 async with self.get_mwman() as mwman: @@ -190,7 +190,7 @@ class TestInvalidOutput(TestManagerBase): req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: - def process_response(self, request, response, spider): + def process_response(self, request, response): return 1 async with self.get_mwman() as mwman: @@ -204,10 +204,10 @@ class TestInvalidOutput(TestManagerBase): req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: - def process_request(self, request, spider): + def process_request(self, request): raise RuntimeError - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return 1 async with self.get_mwman() as mwman: @@ -229,7 +229,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): def cb(self, result): return result - def process_request(self, request, spider): + def process_request(self, request): d = Deferred() d.addCallback(self.cb) d.callback(resp) @@ -252,7 +252,7 @@ class TestMiddlewareUsingCoro(TestManagerBase): download_func = mock.MagicMock() class CoroMiddleware: - async def process_request(self, request, spider): + async def process_request(self, request): await succeed(42) return resp @@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase): download_func = mock.MagicMock() class CoroMiddleware: - async def process_request(self, request, spider): + async def process_request(self, request): await asyncio.sleep(0.1) return await get_from_asyncio_queue(resp) @@ -315,3 +315,41 @@ class TestDownloadDeprecated(TestManagerBase): mwman.download(download_func, req, mwman.crawler.spider) ) assert isinstance(ret, Response) + + +class TestDeprecatedSpiderArg(TestManagerBase): + @deferred_f_from_coro_f + async def test_deprecated_spider_arg(self): + req = Request("http://example.com/index.html") + resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() + + class DeprecatedSpiderArgMiddleware: + def process_request(self, request, spider): + 1 / 0 + + def process_response(self, request, response, spider): + return response + + def process_exception(self, request, exception, spider): + return resp + + async with self.get_mwman() as mwman: + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_request\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_response\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_exception\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgMiddleware()) + result = await maybe_deferred_to_future(mwman.download(download_func, req)) + assert result is resp + assert not download_func.called diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 8bf3a1f09..49215329e 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -8,9 +8,8 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler UNSET = object() @@ -60,9 +59,10 @@ class TestCookiesMiddleware: assert split_cookies(first) == split_cookies(second), msg def setup_method(self): - self.spider = Spider("foo") - self.mw = CookiesMiddleware() - self.redirect_middleware = RedirectMiddleware(settings=Settings()) + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + self.mw = CookiesMiddleware.from_crawler(crawler) + self.redirect_middleware = RedirectMiddleware.from_crawler(crawler) def teardown_method(self): del self.mw @@ -70,15 +70,15 @@ class TestCookiesMiddleware: def test_basic(self): req = Request("http://scrapytest.org/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None assert req2.headers.get("Cookie") == b"C1=value1" def test_setting_false_cookies_enabled(self): @@ -112,9 +112,9 @@ class TestCookiesMiddleware: res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} ) - mw.process_response(req, res, crawler.spider) + mw.process_response(req, res) req2 = Request("http://scrapytest.org/sub1/") - mw.process_request(req2, crawler.spider) + mw.process_request(req2) log.check( ( @@ -143,23 +143,23 @@ class TestCookiesMiddleware: res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} ) - mw.process_response(req, res, crawler.spider) + mw.process_response(req, res) req2 = Request("http://scrapytest.org/sub1/") - mw.process_request(req2, crawler.spider) + mw.process_request(req2) log.check() def test_do_not_break_on_non_utf8_header(self): req = Request("http://scrapytest.org/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None assert "Cookie" in req2.headers def test_dont_merge_cookies(self): @@ -167,11 +167,11 @@ class TestCookiesMiddleware: headers = {"Set-Cookie": "C1=value1; path=/"} req = Request("http://scrapytest.org/") res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res # test Cookie header is not seted to request req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers # check that returned cookies are not merged back to jar @@ -179,16 +179,16 @@ class TestCookiesMiddleware: "http://scrapytest.org/dontmerge", headers={"Set-Cookie": "dont=mergeme; path=/"}, ) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res # check that cookies are merged back req = Request("http://scrapytest.org/mergeme") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers.get("Cookie") == b"C1=value1" # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers.get("Cookie") == b"C1=value1" def test_complex_cookies(self): @@ -216,11 +216,11 @@ class TestCookiesMiddleware: ] req = Request("http://scrapytest.org/", cookies=cookies) - self.mw.process_request(req, self.spider) + self.mw.process_request(req) # embed C1 and C3 for scrapytest.org/foo req = Request("http://scrapytest.org/foo") - self.mw.process_request(req, self.spider) + self.mw.process_request(req) assert req.headers.get("Cookie") in ( b"C1=value1; C3=value3", b"C3=value3; C1=value1", @@ -228,25 +228,25 @@ class TestCookiesMiddleware: # embed C2 for scrapytest.org/bar req = Request("http://scrapytest.org/bar") - self.mw.process_request(req, self.spider) + self.mw.process_request(req) assert req.headers.get("Cookie") == b"C2=value2" # embed nothing for scrapytest.org/baz req = Request("http://scrapytest.org/baz") - self.mw.process_request(req, self.spider) + self.mw.process_request(req) assert "Cookie" not in req.headers def test_merge_request_cookies(self): req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers.get("Cookie"), b"C1=value1; galleta=salada" @@ -258,15 +258,15 @@ class TestCookiesMiddleware: cookies={"galleta": "salada"}, meta={"cookiejar": "store1"}, ) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers, request=req) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/", meta=res.meta) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers.get("Cookie"), b"C1=value1; galleta=salada" ) @@ -276,45 +276,45 @@ class TestCookiesMiddleware: cookies={"galleta": "dulce"}, meta={"cookiejar": "store2"}, ) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None assert req3.headers.get("Cookie") == b"galleta=dulce" headers = {"Set-Cookie": "C2=value2; path=/"} res2 = Response("http://scrapytest.org/", headers=headers, request=req3) - assert self.mw.process_response(req3, res2, self.spider) is res2 + assert self.mw.process_response(req3, res2) is res2 req4 = Request("http://scrapytest.org/", meta=res2.meta) - assert self.mw.process_request(req4, self.spider) is None + assert self.mw.process_request(req4) is None self.assertCookieValEqual( req4.headers.get("Cookie"), b"C2=value2; galleta=dulce" ) # cookies from hosts with port req5_1 = Request("http://scrapytest.org:1104/") - assert self.mw.process_request(req5_1, self.spider) is None + assert self.mw.process_request(req5_1) is None headers = {"Set-Cookie": "C1=value1; path=/"} res5_1 = Response( "http://scrapytest.org:1104/", headers=headers, request=req5_1 ) - assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 + assert self.mw.process_response(req5_1, res5_1) is res5_1 req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") - assert self.mw.process_request(req5_2, self.spider) is None + assert self.mw.process_request(req5_2) is None assert req5_2.headers.get("Cookie") == b"C1=value1" req5_3 = Request("http://scrapytest.org/some-redirected-path") - assert self.mw.process_request(req5_3, self.spider) is None + assert self.mw.process_request(req5_3) is None assert req5_3.headers.get("Cookie") == b"C1=value1" # skip cookie retrieval for not http request req6 = Request("file:///scrapy/sometempfile") - assert self.mw.process_request(req6, self.spider) is None + assert self.mw.process_request(req6) is None assert req6.headers.get("Cookie") is None def test_local_domain(self): request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) - assert self.mw.process_request(request, self.spider) is None + assert self.mw.process_request(request) is None assert "Cookie" in request.headers assert request.headers["Cookie"] == b"currencyCookie=USD" @@ -324,15 +324,15 @@ class TestCookiesMiddleware: mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) - assert mw_default_headers.process_request(req1, self.spider) is None - assert self.mw.process_request(req1, self.spider) is None + assert mw_default_headers.process_request(req1) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual( req1.headers["Cookie"], b"default=something; asdf=qwerty" ) # keep both req2 = Request("http://example.com", cookies={"a": "b"}) - assert mw_default_headers.process_request(req2, self.spider) is None - assert self.mw.process_request(req2, self.spider) is None + assert mw_default_headers.process_request(req2) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty" ) @@ -341,13 +341,13 @@ class TestCookiesMiddleware: def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d") # keep cookies from both 'Cookie' request header and 'cookies' keyword req2 = Request( "http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"} ) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f") # overwrite values from 'Cookie' request header with 'cookies' keyword req3 = Request( @@ -355,40 +355,40 @@ class TestCookiesMiddleware: headers={"Cookie": "a=b; c=d"}, cookies={"a": "new", "e": "f"}, ) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f") def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes req1 = Request("http://example.org", cookies={"a": "á".encode()}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String req3 = Request("http://example.org", cookies={"a": "á"}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String req3 = Request("http://example.org", headers={"Cookie": "a=á"}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") def test_invalid_cookies(self): @@ -402,13 +402,13 @@ class TestCookiesMiddleware: ) as lc: cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] req1 = Request("http://example.org/1", cookies=cookies1) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] req2 = Request("http://example.org/2", cookies=cookies2) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] req3 = Request("http://example.org/3", cookies=cookies3) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None lc.check( ( "scrapy.downloadermiddlewares.cookies", @@ -436,22 +436,22 @@ class TestCookiesMiddleware: def test_primitive_type_cookies(self): # Boolean req1 = Request("http://example.org", cookies={"a": True}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=True") # Float req2 = Request("http://example.org", cookies={"a": 9.5}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5") # Integer req3 = Request("http://example.org", cookies={"a": 10}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=10") # String req4 = Request("http://example.org", cookies={"a": "b"}) - assert self.mw.process_request(req4, self.spider) is None + assert self.mw.process_request(req4) is None self.assertCookieValEqual(req4.headers["Cookie"], b"a=b") def _test_cookie_redirect( @@ -471,7 +471,7 @@ class TestCookiesMiddleware: target.setdefault("status", 301) request1 = Request(cookies=input_cookies, **source) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) cookies = request1.headers.get("Cookie") assert cookies == (b"a=b" if cookies1 else None) @@ -481,16 +481,12 @@ class TestCookiesMiddleware: }, **target, ) - assert self.mw.process_response(request1, response, self.spider) == response + assert self.mw.process_response(request1, response) == response - request2 = self.redirect_middleware.process_response( - request1, - response, - self.spider, - ) + request2 = self.redirect_middleware.process_response(request1, response) assert isinstance(request2, Request) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) cookies = request2.headers.get("Cookie") assert cookies == (b"a=b" if cookies2 else None) @@ -564,11 +560,7 @@ class TestCookiesMiddleware: **target, ) - request2 = self.redirect_middleware.process_response( - request1, - response, - self.spider, - ) + request2 = self.redirect_middleware.process_response(request1, response) assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") @@ -620,12 +612,12 @@ class TestCookiesMiddleware: ] request1 = Request(url1, cookies=input_cookies) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) cookies = request1.headers.get("Cookie") assert cookies == (b"a=b" if cookies1 else None) request2 = Request(url2) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) cookies = request2.headers.get("Cookie") assert cookies == (b"a=b" if cookies2 else None) @@ -674,7 +666,7 @@ class TestCookiesMiddleware: cookies, ): request1 = Request(url1) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) input_cookies = [ { @@ -688,10 +680,10 @@ class TestCookiesMiddleware: "Set-Cookie": _cookies_to_set_cookie_list(input_cookies), } response = Response(url1, status=200, headers=headers) - assert self.mw.process_response(request1, response, self.spider) == response + assert self.mw.process_response(request1, response) == response request2 = Request(url2) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) actual_cookies = request2.headers.get("Cookie") assert actual_cookies == (b"a=b" if cookies else None) @@ -743,7 +735,7 @@ class TestCookiesMiddleware: input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) cookies = request1.headers.get("Cookie") assert cookies == (b"a=b" if cookies1 else None) @@ -752,18 +744,14 @@ class TestCookiesMiddleware: headers={"Location": f"{to_scheme}://a.example"}, status=301, ) - assert self.mw.process_response(request1, response, self.spider) == response + assert self.mw.process_response(request1, response) == response - request2 = self.redirect_middleware.process_response( - request1, - response, - self.spider, - ) + request2 = self.redirect_middleware.process_response(request1, response) assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") assert cookies == (b"a=b" if cookies2 else None) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) cookies = request2.headers.get("Cookie") assert cookies == (b"a=b" if cookies3 else None) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 5716e3631..e697de9b7 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -6,28 +6,27 @@ from scrapy.utils.test import get_crawler class TestDefaultHeadersMiddleware: - def get_defaults_spider_mw(self): + def get_defaults_mw(self): crawler = get_crawler(Spider) - spider = crawler._create_spider("foo") defaults = { to_bytes(k): [to_bytes(v)] for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() } - return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) + return defaults, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): - defaults, spider, mw = self.get_defaults_spider_mw() + defaults, mw = self.get_defaults_mw() req = Request("http://www.scrapytest.org") - mw.process_request(req, spider) + mw.process_request(req) assert req.headers == defaults def test_update_headers(self): - defaults, spider, mw = self.get_defaults_spider_mw() + defaults, mw = self.get_defaults_mw() headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} req = Request("http://www.scrapytest.org", headers=headers) assert req.headers == bytes_headers - mw.process_request(req, spider) + mw.process_request(req) defaults.update(bytes_headers) assert req.headers == defaults diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 31323c8fa..3707cee18 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -14,20 +14,20 @@ class TestDownloadTimeoutMiddleware: def test_default_download_timeout(self): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 180 def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) mw.spider_opened(spider) - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 20.1 def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 2 def test_request_has_download_timeout(self): @@ -35,5 +35,5 @@ class TestDownloadTimeoutMiddleware: spider.download_timeout = 2 mw.spider_opened(spider) req.meta["download_timeout"] = 1 - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 1 diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 9154e1850..522a3002f 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -36,48 +36,48 @@ class TestHttpAuthMiddlewareLegacy: class TestHttpAuthMiddleware: def setup_method(self): self.mw = HttpAuthMiddleware() - self.spider = DomainSpider("foo") - self.mw.spider_opened(self.spider) + spider = DomainSpider("foo") + self.mw.spider_opened(spider) def teardown_method(self): del self.mw def test_no_auth(self): req = Request("http://example-noauth.com/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Authorization" not in req.headers def test_auth_domain(self): req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): req = Request("http://foo.example.com/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers["Authorization"] == b"Digest 123" class TestHttpAuthAnyMiddleware: def setup_method(self): self.mw = HttpAuthMiddleware() - self.spider = AnyDomainSpider("foo") - self.mw.spider_opened(self.spider) + spider = AnyDomainSpider("foo") + self.mw.spider_opened(spider) def teardown_method(self): del self.mw def test_auth(self): req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert req.headers["Authorization"] == b"Digest 123" diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e60d18375..e94591038 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,28 +1,36 @@ +from __future__ import annotations + import email.utils import shutil import tempfile import time from contextlib import contextmanager +from typing import TYPE_CHECKING, Any import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response -from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy.crawler import Crawler + class TestBase: """Base class with common setup and helper methods.""" + policy_class: str + storage_class: str + def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("example.com") self.tmpdir = tempfile.mkdtemp() self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.response = Response( @@ -31,13 +39,11 @@ class TestBase: body=b"test body", status=202, ) - self.crawler.stats.open_spider() def teardown_method(self): - self.crawler.stats.close_spider() shutil.rmtree(self.tmpdir) - def _get_settings(self, **new_settings): + def _get_settings(self, **new_settings: Any) -> dict[str, Any]: settings = { "HTTPCACHE_ENABLED": True, "HTTPCACHE_DIR": self.tmpdir, @@ -47,27 +53,35 @@ class TestBase: "HTTPCACHE_STORAGE": self.storage_class, } settings.update(new_settings) - return Settings(settings) + return settings @contextmanager - def _storage(self, **new_settings): - with self._middleware(**new_settings) as mw: - yield mw.storage - - @contextmanager - def _policy(self, **new_settings): - with self._middleware(**new_settings) as mw: - yield mw.policy - - @contextmanager - def _middleware(self, **new_settings): + def _get_crawler(self, **new_settings: Any) -> Generator[Crawler]: settings = self._get_settings(**new_settings) - mw = HttpCacheMiddleware(settings, self.crawler.stats) - mw.spider_opened(self.spider) + crawler = get_crawler(Spider, settings) + crawler.spider = crawler._create_spider("example.com") + assert crawler.stats + crawler.stats.open_spider() try: - yield mw + yield crawler finally: - mw.spider_closed(self.spider) + crawler.stats.close_spider() + + @contextmanager + def _storage(self, **new_settings: Any): + with self._middleware(**new_settings) as mw: + yield mw.storage, mw.crawler + + @contextmanager + def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]: + with self._get_crawler(**new_settings) as crawler: + assert crawler.spider + mw = HttpCacheMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + try: + yield mw + finally: + mw.spider_closed(crawler.spider) def assertEqualResponse(self, response1, response2): assert response1.url == response2.url @@ -94,37 +108,37 @@ class StorageTestMixin: """Mixin containing storage-specific test methods.""" def test_storage(self): - with self._storage() as storage: + with self._storage() as (storage, crawler): request2 = self.request.copy() - assert storage.retrieve_response(self.spider, request2) is None + assert storage.retrieve_response(crawler.spider, request2) is None - storage.store_response(self.spider, self.request, self.response) - response2 = storage.retrieve_response(self.spider, request2) + storage.store_response(crawler.spider, self.request, self.response) + response2 = storage.retrieve_response(crawler.spider, request2) assert isinstance(response2, HtmlResponse) # content-type header self.assertEqualResponse(self.response, response2) time.sleep(2) # wait for cache to expire - assert storage.retrieve_response(self.spider, request2) is None + assert storage.retrieve_response(crawler.spider, request2) is None def test_storage_never_expire(self): - with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - storage.store_response(self.spider, self.request, self.response) + with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler): + assert storage.retrieve_response(crawler.spider, self.request) is None + storage.store_response(crawler.spider, self.request, self.response) time.sleep(0.5) # give the chance to expire - assert storage.retrieve_response(self.spider, self.request) + assert storage.retrieve_response(crawler.spider, self.request) def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class even if there is no Content-Type header""" - with self._storage() as storage: - assert storage.retrieve_response(self.spider, self.request) is None + with self._storage() as (storage, crawler): + assert storage.retrieve_response(crawler.spider, self.request) is None response = Response( "http://www.example.com", body=b"\n.", status=202, ) - storage.store_response(self.spider, self.request, response) - cached_response = storage.retrieve_response(self.spider, self.request) + storage.store_response(crawler.spider, self.request, response) + cached_response = storage.retrieve_response(crawler.spider, self.request) assert isinstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response) @@ -135,15 +149,15 @@ class PolicyTestMixin: def test_dont_cache(self): with self._middleware() as mw: self.request.meta["dont_cache"] = True - mw.process_response(self.request, self.response, self.spider) - assert mw.storage.retrieve_response(self.spider, self.request) is None + mw.process_response(self.request, self.response) + assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None with self._middleware() as mw: self.request.meta["dont_cache"] = False - mw.process_response(self.request, self.response, self.spider) + mw.process_response(self.request, self.response) if mw.policy.should_cache_response(self.response, self.request): assert isinstance( - mw.storage.retrieve_response(self.spider, self.request), + mw.storage.retrieve_response(mw.crawler.spider, self.request), self.response.__class__, ) @@ -153,9 +167,9 @@ class DummyPolicyTestMixin(PolicyTestMixin): def test_middleware(self): with self._middleware() as mw: - assert mw.process_request(self.request, self.spider) is None - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags @@ -164,9 +178,9 @@ class DummyPolicyTestMixin(PolicyTestMixin): with self._middleware() as mw: req = Request("http://host.com/path") res = Response("http://host2.net/test.html") - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - cached = mw.process_request(req, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) + cached = mw.process_request(req) assert isinstance(cached, Response) self.assertEqualResponse(res, cached) assert "cached" in cached.flags @@ -174,9 +188,9 @@ class DummyPolicyTestMixin(PolicyTestMixin): def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: with pytest.raises(IgnoreRequest): - mw.process_request(self.request, self.spider) - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + mw.process_request(self.request) + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags @@ -185,10 +199,10 @@ class DummyPolicyTestMixin(PolicyTestMixin): # http responses are cached by default req, res = Request("http://test.com/"), Response("http://test.com/") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - cached = mw.process_request(req, self.spider) + cached = mw.process_request(req) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) assert "cached" in cached.flags @@ -196,19 +210,19 @@ class DummyPolicyTestMixin(PolicyTestMixin): # file response is not cached by default req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None # s3 scheme response is cached by default req, res = Request("s3://bucket/key"), Response("http://bucket/key") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - cached = mw.process_request(req, self.spider) + cached = mw.process_request(req) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) assert "cached" in cached.flags @@ -216,25 +230,25 @@ class DummyPolicyTestMixin(PolicyTestMixin): # ignore s3 scheme req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None def test_middleware_ignore_http_codes(self): # test response is not cached with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw: - assert mw.process_request(self.request, self.spider) is None - mw.process_response(self.request, self.response, self.spider) + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None + assert mw.process_request(self.request) is None # test response is cached with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw: - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags @@ -243,14 +257,18 @@ class DummyPolicyTestMixin(PolicyTestMixin): class RFC2616PolicyTestMixin(PolicyTestMixin): """Mixin containing RFC2616 policy specific test methods.""" - def _process_requestresponse(self, mw, request, response): + @staticmethod + def _process_requestresponse( + mw: HttpCacheMiddleware, request: Request, response: Response | None + ) -> Response | Request: result = None try: - result = mw.process_request(request, self.spider) + result = mw.process_request(request) if result: assert isinstance(result, (Request, Response)) return result - result = mw.process_response(request, response, self.spider) + assert response is not None + result = mw.process_response(request, response) assert isinstance(result, Response) return result except Exception: @@ -270,11 +288,11 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): # response for a request with no-store must not be cached res1 = self._process_requestresponse(mw, req1, res0) self.assertEqualResponse(res1, res0) - assert mw.storage.retrieve_response(self.spider, req1) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req1) is None # Re-do request without no-store and expect it to be cached res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags - res3 = mw.process_request(req0, self.spider) + res3 = mw.process_request(req0) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -330,7 +348,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) - resc = mw.storage.retrieve_response(self.spider, req0) + resc = mw.storage.retrieve_response(mw.crawler.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) assert "cached" in res2.flags @@ -354,7 +372,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) - resc = mw.storage.retrieve_response(self.spider, req0) + resc = mw.storage.retrieve_response(mw.crawler.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) assert "cached" in res2.flags @@ -421,7 +439,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): # validate cached response if request max-age set as 0 req1 = req0.replace(headers={"Cache-Control": "max-age=0"}) res304 = res0.replace(status=304) - assert mw.process_request(req1, self.spider) is None + assert mw.process_request(req1) is None res3 = self._process_requestresponse(mw, req1, res304) self.assertEqualResponse(res1, res3) assert "cached" in res3.flags @@ -513,14 +531,14 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): self._process_requestresponse(mw, req0, res0) for e in mw.DOWNLOAD_EXCEPTIONS: # Simulate encountering an error on download attempts - assert mw.process_request(req0, self.spider) is None - res1 = mw.process_exception(req0, e("foo"), self.spider) + assert mw.process_request(req0) is None + res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions - mw.process_request(req0, self.spider) - assert mw.process_exception(req0, Exception("foo"), self.spider) is None + mw.process_request(req0) + assert mw.process_exception(req0, Exception("foo")) is None def test_ignore_response_cache_controls(self): sampledata = [ @@ -578,17 +596,17 @@ class TestDbmStorageWithRFC2616Policy( class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy): dbm_module = "tests.mocks.dummydbm" - def _get_settings(self, **new_settings): + def _get_settings(self, **new_settings) -> dict[str, Any]: new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded - with self._storage() as storage: + with self._storage() as (storage, _): assert storage.dbmodule.__name__ == self.dbm_module class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy): - def _get_settings(self, **new_settings): + def _get_settings(self, **new_settings) -> dict[str, Any]: new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0e58de1f3..65ba2e7e2 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -70,7 +70,6 @@ def _skip_if_no_zstd() -> None: class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider() @@ -124,7 +123,7 @@ class TestHttpCompression: def test_process_request(self): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers - self.mw.process_request(request, self.spider) + self.mw.process_request(request) assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS) def test_process_response_gzip(self): @@ -132,7 +131,7 @@ class TestHttpCompression: request = response.request assert response.headers["Content-Encoding"] == b"gzip" - newresponse = self.mw.process_response(request, response, self.spider) + newresponse = self.mw.process_response(request, response) assert newresponse is not response assert newresponse.body.startswith(b" req.priority def test_dont_redirect(self): @@ -28,7 +29,7 @@ class Base: req = Request(url, meta={"dont_redirect": True}) rsp = self.get_response(req, url2) - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert isinstance(r, Response) assert r is rsp @@ -36,7 +37,7 @@ class Base: req = Request(url, meta={"dont_redirect": False}) rsp = self.get_response(req, url2) - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert isinstance(r, Request) def test_post(self): @@ -50,7 +51,7 @@ class Base: ) rsp = self.get_response(req, url2) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == url2 assert req2.method == "GET" @@ -67,29 +68,29 @@ class Base: req = Request("http://scrapytest.org/302") rsp = self.get_response(req, "/redirected") - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) assert "redirect_times" in req.meta assert req.meta["redirect_times"] == 1 with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp, self.spider) + self.mw.process_response(req, rsp) def test_ttl(self): self.mw.max_redirect_times = 100 req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) rsp = self.get_response(req, "/a") - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp, self.spider) + self.mw.process_response(req, rsp) def test_redirect_urls(self): req1 = Request("http://scrapytest.org/first") rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1, self.spider) + req2 = self.mw.process_response(req1, rsp1) rsp2 = self.get_response(req1, "/redirected2") - req3 = self.mw.process_response(req2, rsp2, self.spider) + req3 = self.mw.process_response(req2, rsp2) assert req2.url == "http://scrapytest.org/redirected" assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"] @@ -102,9 +103,9 @@ class Base: def test_redirect_reasons(self): req1 = Request("http://scrapytest.org/first") rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1, self.spider) + req2 = self.mw.process_response(req1, rsp1) rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2, self.spider) + req3 = self.mw.process_response(req2, rsp2) assert req2.meta["redirect_reasons"] == [self.reason] assert req3.meta["redirect_reasons"] == [self.reason, self.reason] @@ -124,7 +125,7 @@ class Base: original_request, "https://example.com/a" ) internal_redirect_request = self.mw.process_response( - original_request, internal_response, self.spider + original_request, internal_response ) assert isinstance(internal_redirect_request, Request) assert original_request.headers == internal_redirect_request.headers @@ -137,7 +138,7 @@ class Base: ) http_response = self.get_response(http_request, "http://example.com/a") http_redirect_request = self.mw.process_response( - http_request, http_response, self.spider + http_request, http_response ) assert isinstance(http_redirect_request, Request) assert http_request.headers == http_redirect_request.headers @@ -148,7 +149,7 @@ class Base: original_request, "https://example.com:443/a" ) to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response, self.spider + original_request, to_explicit_port_response ) assert isinstance(to_explicit_port_redirect_request, Request) assert original_request.headers == to_explicit_port_redirect_request.headers @@ -159,7 +160,7 @@ class Base: original_request, "https://example.com/a" ) to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response, self.spider + original_request, to_implicit_port_response ) assert isinstance(to_implicit_port_redirect_request, Request) assert original_request.headers == to_implicit_port_redirect_request.headers @@ -171,7 +172,7 @@ class Base: original_request, "https://example.com:8080/a" ) different_port_redirect_request = self.mw.process_response( - original_request, different_port_response, self.spider + original_request, different_port_response ) assert isinstance(different_port_redirect_request, Request) assert { @@ -184,7 +185,7 @@ class Base: original_request, "https://example.org/a" ) external_redirect_request = self.mw.process_response( - original_request, external_response, self.spider + original_request, external_response ) assert isinstance(external_redirect_request, Request) assert safe_headers == external_redirect_request.headers.to_unicode_dict() @@ -194,7 +195,7 @@ class Base: # domain remains the same. upgrade_response = self.get_response(http_request, "https://example.com/a") upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response, self.spider + http_request, upgrade_response ) assert isinstance(upgrade_redirect_request, Request) assert { @@ -215,7 +216,7 @@ class Base: original_request, "http://example.com/a" ) downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response, self.spider + original_request, downgrade_response ) assert isinstance(downgrade_redirect_request, Request) assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() @@ -227,36 +228,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("http://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -269,36 +269,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("http://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -311,36 +310,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("https://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -353,36 +351,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("https://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -395,36 +392,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("http://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -437,36 +433,35 @@ class Base: meta = {"proxy": "https://a:@a.example"} request1 = Request("https://example.com", meta=meta) - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -482,36 +477,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -527,36 +521,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -572,36 +565,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -617,36 +609,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" assert request3.meta["proxy"] == "https://a.example" - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -663,36 +654,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request2.meta["_auth_proxy"] == "https://b.example" assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -708,36 +698,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" assert request1.meta["_auth_proxy"] == "https://a.example" assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" assert request3.meta["_auth_proxy"] == "https://a.example" @@ -753,36 +742,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert "Proxy-Authorization" not in request1.headers assert "_auth_proxy" not in request1.meta assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request2.meta["_auth_proxy"] == "https://b.example" assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta @@ -794,36 +782,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("http://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert "Proxy-Authorization" not in request1.headers assert "_auth_proxy" not in request1.meta assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta @@ -840,36 +827,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request1.meta["_auth_proxy"] == "https://b.example" assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request3.meta["_auth_proxy"] == "https://b.example" @@ -885,36 +871,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request1.meta["_auth_proxy"] == "https://b.example" assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" assert request3.meta["_auth_proxy"] == "https://b.example" @@ -930,36 +915,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert "Proxy-Authorization" not in request1.headers assert "_auth_proxy" not in request1.meta assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" assert request2.meta["_auth_proxy"] == "https://a.example" assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta @@ -971,36 +955,35 @@ class Base: proxy_mw = HttpProxyMiddleware.from_crawler(crawler) request1 = Request("https://example.com") - spider = None - proxy_mw.process_request(request1, spider) + proxy_mw.process_request(request1) assert "Proxy-Authorization" not in request1.headers assert "_auth_proxy" not in request1.meta assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1, spider) + request2 = redirect_mw.process_response(request1, response1) assert isinstance(request2, Request) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta - proxy_mw.process_request(request2, spider) + proxy_mw.process_request(request2) assert "Proxy-Authorization" not in request2.headers assert "_auth_proxy" not in request2.meta assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2, spider) + request3 = redirect_mw.process_response(request2, response2) assert isinstance(request3, Request) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta assert "proxy" not in request3.meta - proxy_mw.process_request(request3, spider) + proxy_mw.process_request(request3) assert "Proxy-Authorization" not in request3.headers assert "_auth_proxy" not in request3.meta @@ -1012,9 +995,9 @@ class TestRedirectMiddleware(Base.Test): reason = 302 def setup_method(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") - self.mw = self.mwcls.from_crawler(self.crawler) + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + self.mw = self.mwcls.from_crawler(crawler) def get_response(self, request, location, status=302): headers = {"Location": location} @@ -1027,14 +1010,14 @@ class TestRedirectMiddleware(Base.Test): req = Request(url, method=method) rsp = Response(url, headers={"Location": url2}, status=status) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == url2 assert req2.method == method # response without Location header but with status code is 3XX should be ignored del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.mw.process_response(req, rsp) is rsp _test("GET") _test("POST") @@ -1054,7 +1037,7 @@ class TestRedirectMiddleware(Base.Test): req = Request(url, method="HEAD") rsp = Response(url, headers={"Location": url2}, status=302) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == url2 assert req2.method == "HEAD" @@ -1066,19 +1049,18 @@ class TestRedirectMiddleware(Base.Test): req = Request(url, method="HEAD") rsp = Response(url, headers={"Location": url2}, status=302) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == url3 assert req2.method == "HEAD" def test_spider_handling(self): - smartspider = self.crawler._create_spider("smarty") - smartspider.handle_httpstatus_list = [404, 301, 302] + self.mw.crawler.spider.handle_httpstatus_list = [404, 301, 302] url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" req = Request(url) rsp = Response(url, headers={"Location": url2}, status=301) - r = self.mw.process_response(req, rsp, smartspider) + r = self.mw.process_response(req, rsp) assert r is rsp def test_request_meta_handling(self): @@ -1087,7 +1069,7 @@ class TestRedirectMiddleware(Base.Test): def _test_passthrough(req): rsp = Response(url, headers={"Location": url2}, status=301, request=req) - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert r is rsp _test_passthrough( @@ -1103,7 +1085,7 @@ class TestRedirectMiddleware(Base.Test): headers={"Location": latin1_location}, status=302, ) - req_result = self.mw.process_response(req, resp, self.spider) + req_result = self.mw.process_response(req, resp) perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o" assert perc_encoded_utf8_url == req_result.url @@ -1115,14 +1097,14 @@ class TestRedirectMiddleware(Base.Test): headers={"Location": utf8_location}, status=302, ) - req_result = self.mw.process_response(req, resp, self.spider) + req_result = self.mw.process_response(req, resp) perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" assert perc_encoded_utf8_url == req_result.url def test_no_location(self): request = Request("https://example.com") response = Response(request.url, status=302) - assert self.mw.process_response(request, response, self.spider) is response + assert self.mw.process_response(request, response) is response SCHEME_PARAMS = ("url", "location", "target") @@ -1166,11 +1148,10 @@ REDIRECT_SCHEME_CASES = ( @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target): crawler = get_crawler(Spider) - spider = crawler._create_spider("foo") mw = RedirectMiddleware.from_crawler(crawler) request = Request(url) response = Response(url, headers={"Location": location}, status=301) - redirect = mw.process_response(request, response, spider) + redirect = mw.process_response(request, response) if target is None: assert redirect == response else: @@ -1189,7 +1170,6 @@ class TestMetaRefreshMiddleware(Base.Test): def setup_method(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider("foo") self.mw = self.mwcls.from_crawler(crawler) def _body(self, interval=5, url="http://example.org/newpage"): @@ -1201,7 +1181,7 @@ class TestMetaRefreshMiddleware(Base.Test): def test_meta_refresh(self): req = Request(url="http://example.org") rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == "http://example.org/newpage" @@ -1211,7 +1191,7 @@ class TestMetaRefreshMiddleware(Base.Test): rsp = HtmlResponse( url="http://example.org", body=self._body(interval=1000), encoding="utf-8" ) - rsp2 = self.mw.process_response(req, rsp, self.spider) + rsp2 = self.mw.process_response(req, rsp) assert rsp is rsp2 def test_meta_refresh_trough_posted_request(self): @@ -1222,7 +1202,7 @@ class TestMetaRefreshMiddleware(Base.Test): headers={"Content-Type": "text/plain", "Content-length": "4"}, ) rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) assert req2.url == "http://example.org/newpage" @@ -1242,7 +1222,7 @@ class TestMetaRefreshMiddleware(Base.Test): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - response = self.mw.process_response(req, rsp, self.spider) + response = self.mw.process_response(req, rsp) assert isinstance(response, Response) def test_ignore_tags_1_x_list(self): @@ -1256,7 +1236,7 @@ class TestMetaRefreshMiddleware(Base.Test): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - response = mw.process_response(req, rsp, self.spider) + response = mw.process_response(req, rsp) assert isinstance(response, Response) @@ -1288,11 +1268,10 @@ class TestMetaRefreshMiddleware(Base.Test): ) def test_meta_refresh_schemes(url, location, target): crawler = get_crawler(Spider) - spider = crawler._create_spider("foo") mw = MetaRefreshMiddleware.from_crawler(crawler) request = Request(url) response = HtmlResponse(url, body=meta_refresh_body(location)) - redirect = mw.process_response(request, response, spider) + redirect = mw.process_response(request, response) if target is None: assert redirect == response else: diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 10fc88026..affcc79c1 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -19,20 +19,21 @@ from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler class TestRetry: def setup_method(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") + self.crawler = get_crawler(DefaultSpider) + self.crawler.spider = self.crawler._create_spider() self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert req2.priority < req.priority def test_404(self): @@ -40,14 +41,14 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/404", body=b"", status=404) # dont retry 404s - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.mw.process_response(req, rsp) is rsp def test_dont_retry(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert r is rsp # Test retry when dont_retry set to False @@ -55,13 +56,13 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503") # first retry - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert r is rsp def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) - r = self.mw.process_exception(req, DNSLookupError(), self.spider) + r = self.mw.process_exception(req, DNSLookupError()) assert r is None def test_503(self): @@ -69,17 +70,17 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) assert req.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) assert req.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.mw.process_response(req, rsp) is rsp assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( @@ -118,7 +119,8 @@ class TestRetry: settings_dict = { "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } - crawler = get_crawler(Spider, settings_dict=settings_dict) + crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) + crawler.spider = crawler._create_spider() mw = RetryMiddleware.from_crawler(crawler) req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) @@ -128,65 +130,61 @@ class TestRetry: mw = self.mw # first retry - req = mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception) assert isinstance(req, Request) assert req.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception) assert isinstance(req, Request) assert req.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception) assert req is None class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_spider_and_middleware(self, settings=None): - crawler = get_crawler(Spider, settings or {}) - spider = crawler._create_spider("foo") - middleware = RetryMiddleware.from_crawler(crawler) - return spider, middleware + def get_middleware(self, settings=None): + crawler = get_crawler(DefaultSpider, settings or {}) + crawler.spider = crawler._create_spider() + return RetryMiddleware.from_crawler(crawler) def test_with_settings_zero(self): max_retry_times = 0 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, DNSLookupError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) def test_with_metakey_zero(self): max_retry_times = 0 - spider, middleware = self.get_spider_and_middleware() + middleware = self.get_middleware() meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( req, DNSLookupError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) def test_without_metakey(self): max_retry_times = 5 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, DNSLookupError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) @@ -198,20 +196,18 @@ class TestMaxRetryTimes: req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) self._test_retry( req1, DNSLookupError("foo"), meta_max_retry_times, - spider=spider, middleware=middleware, ) self._test_retry( req2, DNSLookupError("foo"), middleware_max_retry_times, - spider=spider, middleware=middleware, ) @@ -223,26 +219,24 @@ class TestMaxRetryTimes: req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) self._test_retry( req1, DNSLookupError("foo"), meta_max_retry_times, - spider=spider, middleware=middleware, ) self._test_retry( req2, DNSLookupError("foo"), middleware_max_retry_times, - spider=spider, middleware=middleware, ) def test_with_dont_retry(self): max_retry_times = 4 - spider, middleware = self.get_spider_and_middleware() + middleware = self.get_middleware() meta = { "max_retry_times": max_retry_times, "dont_retry": True, @@ -252,7 +246,6 @@ class TestMaxRetryTimes: req, DNSLookupError("foo"), 0, - spider=spider, middleware=middleware, ) @@ -261,18 +254,16 @@ class TestMaxRetryTimes: req, exception, max_retry_times, - spider=None, middleware=None, ): - spider = spider or self.spider middleware = middleware or self.mw for i in range(max_retry_times): - req = middleware.process_exception(req, exception, spider) + req = middleware.process_exception(req, exception) assert isinstance(req, Request) # discard it - req = middleware.process_exception(req, exception, spider) + req = middleware.process_exception(req, exception) assert req is None diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 3cf570180..bedf40279 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware -from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK @@ -83,10 +82,10 @@ Disallow: /some/randome/page.html async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) d1 = deferred_from_coro( - middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type] + middleware.process_request(Request("http://site.local/allowed1")) ) d2 = deferred_from_coro( - middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type] + middleware.process_request(Request("http://site.local/allowed2")) ) await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) @@ -94,8 +93,8 @@ Disallow: /some/randome/page.html @deferred_f_from_coro_f async def test_robotstxt_multiple_reqs_asyncio(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - c1 = middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type] - c2 = middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type] + c1 = middleware.process_request(Request("http://site.local/allowed1")) + c2 = middleware.process_request(Request("http://site.local/allowed2")) await asyncio.gather(c1, c2) @deferred_f_from_coro_f @@ -164,7 +163,7 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/static/"), middleware) @deferred_f_from_coro_f - async def test_robotstxt_error(self): + async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None: self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -176,9 +175,8 @@ Disallow: /some/randome/page.html self.crawler.engine.download_async.side_effect = return_failure middleware = RobotsTxtMiddleware(self.crawler) - middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - await middleware.process_request(Request("http://site.local"), None) - assert middleware._logerror.called + await middleware.process_request(Request("http://site.local")) + assert "DNS lookup failed: Robotstxt address not found" in caplog.text @deferred_f_from_coro_f async def test_robotstxt_immediate_error(self): @@ -205,10 +203,13 @@ Disallow: /some/randome/page.html self.crawler.engine.download_async.side_effect = ignore_request middleware = RobotsTxtMiddleware(self.crawler) - mw_module_logger.error = mock.MagicMock() - - await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - assert not mw_module_logger.error.called # type: ignore[attr-defined] + with mock.patch( + "scrapy.downloadermiddlewares.robotstxt.logger" + ) as mw_module_logger: + await self.assertNotIgnored( + Request("http://site.local/allowed"), middleware + ) + assert not mw_module_logger.error.called def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() @@ -216,7 +217,7 @@ Disallow: /some/randome/page.html crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)") middleware = RobotsTxtMiddleware(crawler) rp = mock.MagicMock(return_value=True) - middleware.process_request_2(rp, Request("http://site.local/allowed"), None) + middleware.process_request_2(rp, Request("http://site.local/allowed")) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") @deferred_f_from_coro_f @@ -224,34 +225,30 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) middleware.process_request_2 = mock.MagicMock() - await middleware.process_request( - Request("data:text/plain,Hello World data"), None - ) + await middleware.process_request(Request("data:text/plain,Hello World data")) assert not middleware.process_request_2.called await middleware.process_request( - Request("file:///tests/sample_data/test_site/nothinghere.html"), None + Request("file:///tests/sample_data/test_site/nothinghere.html") ) assert not middleware.process_request_2.called - await middleware.process_request(Request("http://site.local/allowed"), None) + await middleware.process_request(Request("http://site.local/allowed")) assert middleware.process_request_2.called async def assertNotIgnored( self, request: Request, middleware: RobotsTxtMiddleware ) -> None: - spider = None # not actually used try: - await middleware.process_request(request, spider) # type: ignore[arg-type] + await middleware.process_request(request) except IgnoreRequest: pytest.fail("IgnoreRequest was raised unexpectedly") async def assertIgnored( self, request: Request, middleware: RobotsTxtMiddleware ) -> None: - spider = None # not actually used with pytest.raises(IgnoreRequest): - await middleware.process_request(request, spider) # type: ignore[arg-type] + await middleware.process_request(request) def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download_async.call_args_list diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 2f1c97a34..67af4264c 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -11,7 +11,6 @@ class MyException(Exception): class TestDownloaderStats: def setup_method(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider() @@ -25,15 +24,15 @@ class TestDownloaderStats: ) def test_process_request(self): - self.mw.process_request(self.req, self.spider) + self.mw.process_request(self.req) self.assertStatsEqual("downloader/request_count", 1) def test_process_response(self): - self.mw.process_response(self.req, self.res, self.spider) + self.mw.process_response(self.req, self.res) self.assertStatsEqual("downloader/response_count", 1) def test_process_exception(self): - self.mw.process_exception(self.req, MyException(), self.spider) + self.mw.process_exception(self.req, MyException()) self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( "downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException", diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 1497f8c67..60dc2ae7a 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -11,9 +11,9 @@ class TestUserAgentMiddleware: return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): - spider, mw = self.get_spider_and_mw("default_useragent") + _, mw = self.get_spider_and_mw("default_useragent") req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.headers["User-Agent"] == b"default_useragent" def test_remove_agent(self): @@ -22,7 +22,7 @@ class TestUserAgentMiddleware: spider.user_agent = None mw.spider_opened(spider) req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.headers.get("User-Agent") is None def test_spider_agent(self): @@ -30,7 +30,7 @@ class TestUserAgentMiddleware: spider.user_agent = "spider_useragent" mw.spider_opened(spider) req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.headers["User-Agent"] == b"spider_useragent" def test_header_agent(self): @@ -40,7 +40,7 @@ class TestUserAgentMiddleware: req = Request( "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} ) - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert req.headers["User-Agent"] == b"header_useragent" def test_no_agent(self): @@ -48,5 +48,5 @@ class TestUserAgentMiddleware: spider.user_agent = None mw.spider_opened(spider) req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert "User-Agent" not in req.headers diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 6c3008b25..f3d619d3c 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -20,7 +20,7 @@ class M1: def close_spider(self, spider): pass - def process(self, response, request, spider): + def process(self, response, request): pass @@ -33,7 +33,7 @@ class M2: class M3: - def process(self, response, request, spider): + def process(self, response, request): pass diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index de0f0d895..e2c1668d8 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -11,18 +11,18 @@ OVERRIDDEN_URL = "https://example.org" class ProcessResponseMiddleware: - def process_response(self, request, response, spider): + def process_response(self, request, response): return response.replace(request=Request(OVERRIDDEN_URL)) class RaiseExceptionRequestMiddleware: - def process_request(self, request, spider): + def process_request(self, request): 1 / 0 return request class CatchExceptionOverrideRequestMiddleware: - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), @@ -31,7 +31,7 @@ class CatchExceptionOverrideRequestMiddleware: class CatchExceptionDoNotOverrideRequestMiddleware: - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), @@ -46,10 +46,17 @@ class AlternativeCallbacksSpider(SingleRequestSpider): class AlternativeCallbacksMiddleware: - def process_response(self, request, response, spider): + def __init__(self, crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def process_response(self, request, response): new_request = request.replace( url=OVERRIDDEN_URL, - callback=spider.alt_callback, + callback=self.crawler.spider.alt_callback, cb_kwargs={"foo": "bar"}, ) return response.replace(request=new_request) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 0f31af8ad..67ad4ba1e 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -12,11 +12,11 @@ class InjectArgumentsDownloaderMiddleware: Make sure downloader middlewares are able to update the keyword arguments """ - def process_request(self, request, spider): + def process_request(self, request): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_request"] = True - def process_response(self, request, response, spider): + def process_response(self, request, response): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_response"] = True return response diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 658351999..68fcad98c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -31,15 +31,13 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) -from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler if TYPE_CHECKING: from collections.abc import Sequence - -@pytest.fixture -def spider() -> Spider: - return Spider("foo") + from scrapy.crawler import Crawler class TestRefererMiddleware: @@ -1011,20 +1009,24 @@ class TestReferrerOnRedirect(TestRefererMiddleware): ] @pytest.fixture - def referrermw(self) -> RefererMiddleware: - settings = Settings(self.settings) - return RefererMiddleware(settings) + def crawler(self) -> Crawler: + crawler = get_crawler(DefaultSpider, self.settings) + crawler.spider = crawler._create_spider() + return crawler @pytest.fixture - def redirectmw(self) -> RedirectMiddleware: - settings = Settings(self.settings) - return RedirectMiddleware(settings) + def referrermw(self, crawler: Crawler) -> RefererMiddleware: + return RefererMiddleware.from_crawler(crawler) + + @pytest.fixture + def redirectmw(self, crawler: Crawler) -> RedirectMiddleware: + return RedirectMiddleware.from_crawler(crawler) def test( # type: ignore[override] self, + crawler: Crawler, referrermw: RefererMiddleware, redirectmw: RedirectMiddleware, - spider: Spider, ) -> None: for ( parent, @@ -1044,9 +1046,10 @@ class TestReferrerOnRedirect(TestRefererMiddleware): request.url, headers={"Location": url}, status=status ) request = cast( - "Request", redirectmw.process_response(request, response, spider) + "Request", redirectmw.process_response(request, response) ) - referrermw.request_scheduled(request, spider) + assert crawler.spider + referrermw.request_scheduled(request, crawler.spider) assert isinstance(request, Request) assert request.headers.get("Referer") == final_referrer