RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate the spider arg of downloader middleware methods. (#7037) 

* Deprecate the spider arg of downloader middleware methods.

* Add a test for deprecated spider args in downloader mws.
This commit is contained in:
Andrey Rakhmatullin 2025-09-02 20:51:47 +04:00 committed by GitHub
parent c097921c44
commit 2ad81a0ef8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 728 additions and 665 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/topics/downloader-middleware.rst
View File

@ -70,7 +70,7 @@ defines one or more of these methods:
.. note:: Any of the downloader middleware methods may also return a deferred.
.. method:: process_request(request, spider)
.. method:: process_request(request)
This method is called for each request that goes through the download
middleware.
@ -102,10 +102,7 @@ defines one or more of these methods:
:param request: the request being processed
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.Spider` object
.. method:: process_response(request, response, spider)
.. method:: process_response(request, response)
:meth:`process_response` should either: return a :class:`~scrapy.http.Response`
object, return a :class:`~scrapy.Request` object or
@ -129,10 +126,7 @@ defines one or more of these methods:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.Spider` object
.. method:: process_exception(request, exception, spider)
.. method:: process_exception(request, exception)
Scrapy calls :meth:`process_exception` when a download handler
or a :meth:`process_request` (from a downloader middleware) raises an
@ -160,9 +154,6 @@ defines one or more of these methods:
:param exception: the raised exception
:type exception: an ``Exception`` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.Spider` object
.. _topics-downloader-middleware-ref:
Built-in downloader middleware reference

36
scrapy/core/downloader/middleware.py
View File

@ -35,10 +35,13 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "process_request"):
self.methods["process_request"].append(mw.process_request)
self._check_mw_method_spider_arg(mw.process_request)
if hasattr(mw, "process_response"):
self.methods["process_response"].appendleft(mw.process_response)
self._check_mw_method_spider_arg(mw.process_response)
if hasattr(mw, "process_exception"):
self.methods["process_exception"].appendleft(mw.process_exception)
self._check_mw_method_spider_arg(mw.process_exception)
@inlineCallbacks
def download(
@ -64,9 +67,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_request"]:
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, spider=self._spider)
)
if method in self._mw_methods_requiring_spider:
response = yield deferred_from_coro(
method(request=request, spider=self._spider)
)
else:
response = yield deferred_from_coro(method(request=request))
if response is not None and not isinstance(
response, (Response, Request)
):
@ -91,9 +97,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods["process_response"]:
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, response=response, spider=self._spider)
)
if method in self._mw_methods_requiring_spider:
response = yield deferred_from_coro(
method(request=request, response=response, spider=self._spider)
)
else:
response = yield deferred_from_coro(
method(request=request, response=response)
)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, "
@ -109,9 +120,16 @@ class DownloaderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_exception"]:
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, exception=exception, spider=self._spider)
)
if method in self._mw_methods_requiring_spider:
response = yield deferred_from_coro(
method(
request=request, exception=exception, spider=self._spider
)
)
else:
response = yield deferred_from_coro(
method(request=request, exception=exception)
)
if response is not None and not isinstance(
response, (Response, Request)
):

25
scrapy/downloadermiddlewares/cookies.py
View File

@ -9,6 +9,7 @@ from tldextract import TLDExtract
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@ -39,6 +40,8 @@ def _is_public_domain(domain: str) -> bool:
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
crawler: Crawler
def __init__(self, debug: bool = False):
self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar)
self.debug: bool = debug
@ -47,7 +50,9 @@ class CookiesMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured
return cls(crawler.settings.getbool("COOKIES_DEBUG"))
o = cls(crawler.settings.getbool("COOKIES_DEBUG"))
o.crawler = crawler
return o
def _process_cookies(
self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
@ -67,8 +72,9 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request)
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
if request.meta.get("dont_merge_cookies", False):
return None
@ -81,11 +87,12 @@ class CookiesMiddleware:
# set Cookie header
request.headers.pop("Cookie", None)
jar.add_cookie_header(request)
self._debug_cookie(request, spider)
self._debug_cookie(request)
return None
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if request.meta.get("dont_merge_cookies", False):
return response
@ -96,11 +103,11 @@ class CookiesMiddleware:
cookies = jar.make_cookies(response, request)
self._process_cookies(cookies, jar=jar, request=request)
self._debug_set_cookie(response, spider)
self._debug_set_cookie(response)
return response
def _debug_cookie(self, request: Request, spider: Spider) -> None:
def _debug_cookie(self, request: Request) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -109,9 +116,9 @@ class CookiesMiddleware:
if cl:
cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
logger.debug(msg, extra={"spider": self.crawler.spider})
def _debug_set_cookie(self, response: Response, spider: Spider) -> None:
def _debug_set_cookie(self, response: Response) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -120,7 +127,7 @@ class CookiesMiddleware:
if cl:
cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
logger.debug(msg, extra={"spider": self.crawler.spider})
def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None:
"""

4
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -8,6 +8,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
@ -30,8 +31,9 @@ class DefaultHeadersMiddleware:
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items())
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
for k, v in self._headers:
request.headers.setdefault(k, v)

4
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -9,6 +9,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -31,8 +32,9 @@ class DownloadTimeoutMiddleware:
def spider_opened(self, spider: Spider) -> None:
self._timeout = getattr(spider, "download_timeout", self._timeout)
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
if self._timeout:
request.meta.setdefault("download_timeout", self._timeout)

4
scrapy/downloadermiddlewares/httpauth.py
View File

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from w3lib.http import basic_auth_header
from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
@ -38,8 +39,9 @@ class HttpAuthMiddleware:
self.auth = basic_auth_header(usr, pwd)
self.domain = spider.http_auth_domain # type: ignore[attr-defined]
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
auth = getattr(self, "auth", None)
if (

29
scrapy/downloadermiddlewares/httpcache.py
View File

@ -17,6 +17,7 @@ from twisted.web.client import ResponseFailed
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
@ -45,6 +46,8 @@ class HttpCacheMiddleware:
OSError,
)
crawler: Crawler
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool("HTTPCACHE_ENABLED"):
raise NotConfigured
@ -59,6 +62,7 @@ class HttpCacheMiddleware:
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
o.crawler = crawler
return o
def spider_opened(self, spider: Spider) -> None:
@ -67,8 +71,9 @@ class HttpCacheMiddleware:
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
if request.meta.get("dont_cache", False):
return None
@ -80,7 +85,7 @@ class HttpCacheMiddleware:
# Look for cached response and check if expired
cachedresponse: Response | None = self.storage.retrieve_response(
spider, request
self.crawler.spider, request
)
if cachedresponse is None:
self.stats.inc_value("httpcache/miss")
@ -101,8 +106,9 @@ class HttpCacheMiddleware:
return None
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if request.meta.get("dont_cache", False):
return response
@ -121,7 +127,7 @@ class HttpCacheMiddleware:
cachedresponse: Response | None = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand")
self._cache_response(spider, response, request, cachedresponse)
self._cache_response(response, request)
return response
if self.policy.is_cached_response_valid(cachedresponse, response, request):
@ -129,11 +135,12 @@ class HttpCacheMiddleware:
return cachedresponse
self.stats.inc_value("httpcache/invalidate")
self._cache_response(spider, response, request, cachedresponse)
self._cache_response(response, request)
return response
@_warn_spider_arg
def process_exception(
self, request: Request, exception: Exception, spider: Spider
self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None:
cachedresponse: Response | None = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance(
@ -143,15 +150,9 @@ class HttpCacheMiddleware:
return cachedresponse
return None
def _cache_response(
self,
spider: Spider,
response: Response,
request: Request,
cachedresponse: Response | None,
) -> None:
def _cache_response(self, response: Response, request: Request) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value("httpcache/store")
self.storage.store_response(spider, request, response)
self.storage.store_response(self.crawler.spider, request, response)
else:
self.stats.inc_value("httpcache/uncacheable")

7
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -15,6 +15,7 @@ from scrapy.utils._compression import (
_unbrotli,
_unzstd,
)
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.gz import gunzip
if TYPE_CHECKING:
@ -93,14 +94,16 @@ class HttpCompressionMiddleware:
)
self._warn_size = spider.download_warnsize
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
return None
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if request.method == "HEAD":
return response

4
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -10,6 +10,7 @@ from urllib.request import ( # type: ignore[attr-defined]
)
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
@ -55,8 +56,9 @@ class HttpProxyMiddleware:
return creds, proxy_url
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
creds, proxy_url, scheme = None, None, None
if "proxy" in request.meta:

22
scrapy/downloadermiddlewares/offsite.py
View File

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
@ -21,29 +22,34 @@ logger = logging.getLogger(__name__)
class OffsiteMiddleware:
crawler: Crawler
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen: set[str] = set()
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
o.crawler = crawler
return o
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen: set[str] = set()
def spider_opened(self, spider: Spider) -> None:
self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
def request_scheduled(self, request: Request, spider: Spider) -> None:
self.process_request(request, spider)
self.process_request(request)
def process_request(self, request: Request, spider: Spider) -> None:
@_warn_spider_arg
def process_request(self, request: Request, spider: Spider | None = None) -> None:
assert self.crawler.spider
if (
request.dont_filter
or request.meta.get("allow_offsite")
or self.should_follow(request, spider)
or self.should_follow(request, self.crawler.spider)
):
return
domain = urlparse_cached(request).hostname
@ -52,7 +58,7 @@ class OffsiteMiddleware:
logger.debug(
"Filtered offsite request to %(domain)r: %(request)s",
{"domain": domain, "request": request},
extra={"spider": spider},
extra={"spider": self.crawler.spider},
)
self.stats.inc_value("offsite/domains")
self.stats.inc_value("offsite/filtered")

29
scrapy/downloadermiddlewares/redirect.py
View File

@ -8,6 +8,7 @@ from w3lib.url import safe_url_string
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Response
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
@ -79,6 +80,7 @@ def _build_redirect_request(
class BaseRedirectMiddleware:
crawler: Crawler
enabled_setting: str = "REDIRECT_ENABLED"
def __init__(self, settings: BaseSettings):
@ -90,11 +92,11 @@ class BaseRedirectMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
o = cls(crawler.settings)
o.crawler = crawler
return o
def _redirect(
self, redirected: Request, request: Request, spider: Spider, reason: Any
) -> Request:
def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request:
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1
@ -114,13 +116,13 @@ class BaseRedirectMiddleware:
logger.debug(
"Redirecting (%(reason)s) to %(redirected)s from %(request)s",
{"reason": reason, "redirected": redirected, "request": request},
extra={"spider": spider},
extra={"spider": self.crawler.spider},
)
return redirected
logger.debug(
"Discarding %(request)s: max redirections reached",
{"request": request},
extra={"spider": spider},
extra={"spider": self.crawler.spider},
)
raise IgnoreRequest("max redirections reached")
@ -144,12 +146,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
and meta-refresh html tag.
"""
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if (
request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", [])
or response.status
in getattr(self.crawler.spider, "handle_httpstatus_list", [])
or response.status in request.meta.get("handle_httpstatus_list", [])
or request.meta.get("handle_httpstatus_all", False)
):
@ -171,10 +175,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
return response
if response.status in (301, 307, 308) or request.method == "HEAD":
return self._redirect(redirected, request, spider, response.status)
return self._redirect(redirected, request, response.status)
redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status)
return self._redirect(redirected, request, response.status)
class MetaRefreshMiddleware(BaseRedirectMiddleware):
@ -185,8 +189,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if (
request.meta.get("dont_redirect", False)
@ -203,5 +208,5 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
if cast("float", interval) < self._maxdelay:
return self._redirect(redirected, request, spider, "meta refresh")
return self._redirect(redirected, request, "meta refresh")
return response

25
scrapy/downloadermiddlewares/retry.py
View File

@ -16,6 +16,7 @@ from logging import Logger, getLogger
from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@ -123,6 +124,8 @@ def get_retry_request(
class RetryMiddleware:
crawler: Crawler
def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
@ -136,39 +139,41 @@ class RetryMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
o = cls(crawler.settings)
o.crawler = crawler
return o
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if request.meta.get("dont_retry", False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
return self._retry(request, reason) or response
return response
@_warn_spider_arg
def process_exception(
self, request: Request, exception: Exception, spider: Spider
self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False
):
return self._retry(request, exception, spider)
return self._retry(request, exception)
return None
def _retry(
self,
request: Request,
reason: str | Exception | type[Exception],
spider: Spider,
self, request: Request, reason: str | Exception | type[Exception]
) -> Request | None:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
assert self.crawler.spider
return get_retry_request(
request,
reason=reason,
spider=spider,
spider=self.crawler.spider,
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)

37
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -14,6 +14,7 @@ from twisted.internet.defer import Deferred
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@ -51,17 +52,18 @@ class RobotsTxtMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
async def process_request(self, request: Request, spider: Spider) -> None:
@_warn_spider_arg
async def process_request(
self, request: Request, spider: Spider | None = None
) -> None:
if request.meta.get("dont_obey_robotstxt"):
return
if request.url.startswith("data:") or request.url.startswith("file:"):
return
rp = await self.robot_parser(request, spider)
self.process_request_2(rp, request, spider)
rp = await self.robot_parser(request)
self.process_request_2(rp, request)
def process_request_2(
self, rp: RobotParser | None, request: Request, spider: Spider
) -> None:
def process_request_2(self, rp: RobotParser | None, request: Request) -> None:
if rp is None:
return
@ -73,15 +75,13 @@ class RobotsTxtMiddleware:
logger.debug(
"Forbidden by robots.txt: %(request)s",
{"request": request},
extra={"spider": spider},
extra={"spider": self.crawler.spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
async def robot_parser(
self, request: Request, spider: Spider
) -> RobotParser | None:
async def robot_parser(self, request: Request) -> RobotParser | None:
url = urlparse_cached(request)
netloc = url.netloc
@ -100,7 +100,13 @@ class RobotsTxtMiddleware:
resp = await self.crawler.engine.download_async(robotsreq)
self._parse_robots(resp, netloc)
except Exception as e:
self._logerror(e, robotsreq, spider)
if not isinstance(e, IgnoreRequest):
logger.error(
"Error downloading %(request)s: %(f_exception)s",
{"request": request, "f_exception": e},
exc_info=True,
extra={"spider": self.crawler.spider},
)
self._robots_error(e, netloc)
self.crawler.stats.inc_value("robotstxt/request_count")
@ -109,15 +115,6 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser)
return parser
def _logerror(self, exc: Exception, request: Request, spider: Spider) -> None:
if not isinstance(exc, IgnoreRequest):
logger.error(
"Error downloading %(request)s: %(f_exception)s",
{"request": request, "f_exception": exc},
exc_info=True, # noqa: LOG014
extra={"spider": spider},
)
def _parse_robots(self, response: Response, netloc: str) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")

10
scrapy/downloadermiddlewares/stats.py
View File

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from twisted.web import http
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
@ -45,8 +46,9 @@ class DownloaderStats:
assert crawler.stats
return cls(crawler.stats)
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
self.stats.inc_value("downloader/request_count")
self.stats.inc_value(f"downloader/request_method_count/{request.method}")
@ -54,8 +56,9 @@ class DownloaderStats:
self.stats.inc_value("downloader/request_bytes", reqlen)
return None
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
self.stats.inc_value("downloader/response_count")
self.stats.inc_value(f"downloader/response_status_count/{response.status}")
@ -69,8 +72,9 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_bytes", reslen)
return response
@_warn_spider_arg
def process_exception(
self, request: Request, exception: Exception, spider: Spider
self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None:
ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count")

4
scrapy/downloadermiddlewares/useragent.py
View File

@ -5,6 +5,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -29,8 +30,9 @@ class UserAgentMiddleware:
def spider_opened(self, spider: Spider) -> None:
self.user_agent = getattr(spider, "user_agent", self.user_agent)
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
if self.user_agent:
request.headers.setdefault(b"User-Agent", self.user_agent)

58
tests/test_downloadermiddleware.py
View File

@ -128,7 +128,7 @@ class TestResponseFromProcessRequest(TestManagerBase):
download_func = mock.MagicMock()
class ResponseMiddleware:
def process_request(self, request, spider):
def process_request(self, request):
return resp
async with self.get_mwman() as mwman:
@ -151,11 +151,11 @@ class TestResponseFromProcessException(TestManagerBase):
raise ValueError("test")
class ResponseMiddleware:
def process_response(self, request, response, spider):
def process_response(self, request, response):
calls.append("process_response")
return resp
def process_exception(self, request, exception, spider):
def process_exception(self, request, exception):
calls.append("process_exception")
return resp
@ -176,7 +176,7 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html")
class InvalidProcessRequestMiddleware:
def process_request(self, request, spider):
def process_request(self, request):
return 1
async with self.get_mwman() as mwman:
@ -190,7 +190,7 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html")
class InvalidProcessResponseMiddleware:
def process_response(self, request, response, spider):
def process_response(self, request, response):
return 1
async with self.get_mwman() as mwman:
@ -204,10 +204,10 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html")
class InvalidProcessExceptionMiddleware:
def process_request(self, request, spider):
def process_request(self, request):
raise RuntimeError
def process_exception(self, request, exception, spider):
def process_exception(self, request, exception):
return 1
async with self.get_mwman() as mwman:
@ -229,7 +229,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
def cb(self, result):
return result
def process_request(self, request, spider):
def process_request(self, request):
d = Deferred()
d.addCallback(self.cb)
d.callback(resp)
@ -252,7 +252,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
download_func = mock.MagicMock()
class CoroMiddleware:
async def process_request(self, request, spider):
async def process_request(self, request):
await succeed(42)
return resp
@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
download_func = mock.MagicMock()
class CoroMiddleware:
async def process_request(self, request, spider):
async def process_request(self, request):
await asyncio.sleep(0.1)
return await get_from_asyncio_queue(resp)
@ -315,3 +315,41 @@ class TestDownloadDeprecated(TestManagerBase):
mwman.download(download_func, req, mwman.crawler.spider)
)
assert isinstance(ret, Response)
class TestDeprecatedSpiderArg(TestManagerBase):
@deferred_f_from_coro_f
async def test_deprecated_spider_arg(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
download_func = mock.MagicMock()
class DeprecatedSpiderArgMiddleware:
def process_request(self, request, spider):
1 / 0
def process_response(self, request, response, spider):
return response
def process_exception(self, request, exception, spider):
return resp
async with self.get_mwman() as mwman:
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_request\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_response\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_exception\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgMiddleware())
result = await maybe_deferred_to_future(mwman.download(download_func, req))
assert result is resp
assert not download_func.called

158
tests/test_downloadermiddleware_cookies.py
View File

@ -8,9 +8,8 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
UNSET = object()
@ -60,9 +59,10 @@ class TestCookiesMiddleware:
assert split_cookies(first) == split_cookies(second), msg
def setup_method(self):
self.spider = Spider("foo")
self.mw = CookiesMiddleware()
self.redirect_middleware = RedirectMiddleware(settings=Settings())
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
self.mw = CookiesMiddleware.from_crawler(crawler)
self.redirect_middleware = RedirectMiddleware.from_crawler(crawler)
def teardown_method(self):
del self.mw
@ -70,15 +70,15 @@ class TestCookiesMiddleware:
def test_basic(self):
req = Request("http://scrapytest.org/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers
headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
assert req2.headers.get("Cookie") == b"C1=value1"
def test_setting_false_cookies_enabled(self):
@ -112,9 +112,9 @@ class TestCookiesMiddleware:
res = Response(
"http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
)
mw.process_response(req, res, crawler.spider)
mw.process_response(req, res)
req2 = Request("http://scrapytest.org/sub1/")
mw.process_request(req2, crawler.spider)
mw.process_request(req2)
log.check(
(
@ -143,23 +143,23 @@ class TestCookiesMiddleware:
res = Response(
"http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
)
mw.process_response(req, res, crawler.spider)
mw.process_response(req, res)
req2 = Request("http://scrapytest.org/sub1/")
mw.process_request(req2, crawler.spider)
mw.process_request(req2)
log.check()
def test_do_not_break_on_non_utf8_header(self):
req = Request("http://scrapytest.org/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers
headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"}
res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
assert "Cookie" in req2.headers
def test_dont_merge_cookies(self):
@ -167,11 +167,11 @@ class TestCookiesMiddleware:
headers = {"Set-Cookie": "C1=value1; path=/"}
req = Request("http://scrapytest.org/")
res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
# test Cookie header is not seted to request
req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1})
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers
# check that returned cookies are not merged back to jar
@ -179,16 +179,16 @@ class TestCookiesMiddleware:
"http://scrapytest.org/dontmerge",
headers={"Set-Cookie": "dont=mergeme; path=/"},
)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
# check that cookies are merged back
req = Request("http://scrapytest.org/mergeme")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"C1=value1"
# check that cookies are merged when dont_merge_cookies is passed as 0
req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0})
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"C1=value1"
def test_complex_cookies(self):
@ -216,11 +216,11 @@ class TestCookiesMiddleware:
]
req = Request("http://scrapytest.org/", cookies=cookies)
self.mw.process_request(req, self.spider)
self.mw.process_request(req)
# embed C1 and C3 for scrapytest.org/foo
req = Request("http://scrapytest.org/foo")
self.mw.process_request(req, self.spider)
self.mw.process_request(req)
assert req.headers.get("Cookie") in (
b"C1=value1; C3=value3",
b"C3=value3; C1=value1",
@ -228,25 +228,25 @@ class TestCookiesMiddleware:
# embed C2 for scrapytest.org/bar
req = Request("http://scrapytest.org/bar")
self.mw.process_request(req, self.spider)
self.mw.process_request(req)
assert req.headers.get("Cookie") == b"C2=value2"
# embed nothing for scrapytest.org/baz
req = Request("http://scrapytest.org/baz")
self.mw.process_request(req, self.spider)
self.mw.process_request(req)
assert "Cookie" not in req.headers
def test_merge_request_cookies(self):
req = Request("http://scrapytest.org/", cookies={"galleta": "salada"})
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"galleta=salada"
headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(
req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
@ -258,15 +258,15 @@ class TestCookiesMiddleware:
cookies={"galleta": "salada"},
meta={"cookiejar": "store1"},
)
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"galleta=salada"
headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers, request=req)
assert self.mw.process_response(req, res, self.spider) is res
assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/", meta=res.meta)
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(
req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
)
@ -276,45 +276,45 @@ class TestCookiesMiddleware:
cookies={"galleta": "dulce"},
meta={"cookiejar": "store2"},
)
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
assert req3.headers.get("Cookie") == b"galleta=dulce"
headers = {"Set-Cookie": "C2=value2; path=/"}
res2 = Response("http://scrapytest.org/", headers=headers, request=req3)
assert self.mw.process_response(req3, res2, self.spider) is res2
assert self.mw.process_response(req3, res2) is res2
req4 = Request("http://scrapytest.org/", meta=res2.meta)
assert self.mw.process_request(req4, self.spider) is None
assert self.mw.process_request(req4) is None
self.assertCookieValEqual(
req4.headers.get("Cookie"), b"C2=value2; galleta=dulce"
)
# cookies from hosts with port
req5_1 = Request("http://scrapytest.org:1104/")
assert self.mw.process_request(req5_1, self.spider) is None
assert self.mw.process_request(req5_1) is None
headers = {"Set-Cookie": "C1=value1; path=/"}
res5_1 = Response(
"http://scrapytest.org:1104/", headers=headers, request=req5_1
)
assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1
assert self.mw.process_response(req5_1, res5_1) is res5_1
req5_2 = Request("http://scrapytest.org:1104/some-redirected-path")
assert self.mw.process_request(req5_2, self.spider) is None
assert self.mw.process_request(req5_2) is None
assert req5_2.headers.get("Cookie") == b"C1=value1"
req5_3 = Request("http://scrapytest.org/some-redirected-path")
assert self.mw.process_request(req5_3, self.spider) is None
assert self.mw.process_request(req5_3) is None
assert req5_3.headers.get("Cookie") == b"C1=value1"
# skip cookie retrieval for not http request
req6 = Request("file:///scrapy/sometempfile")
assert self.mw.process_request(req6, self.spider) is None
assert self.mw.process_request(req6) is None
assert req6.headers.get("Cookie") is None
def test_local_domain(self):
request = Request("http://example-host/", cookies={"currencyCookie": "USD"})
assert self.mw.process_request(request, self.spider) is None
assert self.mw.process_request(request) is None
assert "Cookie" in request.headers
assert request.headers["Cookie"] == b"currencyCookie=USD"
@ -324,15 +324,15 @@ class TestCookiesMiddleware:
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
# overwrite with values from 'cookies' request argument
req1 = Request("http://example.org", cookies={"default": "something"})
assert mw_default_headers.process_request(req1, self.spider) is None
assert self.mw.process_request(req1, self.spider) is None
assert mw_default_headers.process_request(req1) is None
assert self.mw.process_request(req1) is None
self.assertCookieValEqual(
req1.headers["Cookie"], b"default=something; asdf=qwerty"
)
# keep both
req2 = Request("http://example.com", cookies={"a": "b"})
assert mw_default_headers.process_request(req2, self.spider) is None
assert self.mw.process_request(req2, self.spider) is None
assert mw_default_headers.process_request(req2) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(
req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty"
)
@ -341,13 +341,13 @@ class TestCookiesMiddleware:
def test_keep_cookie_header(self):
# keep only cookies from 'Cookie' request header
req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"})
assert self.mw.process_request(req1, self.spider) is None
assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d")
# keep cookies from both 'Cookie' request header and 'cookies' keyword
req2 = Request(
"http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"}
)
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f")
# overwrite values from 'Cookie' request header with 'cookies' keyword
req3 = Request(
@ -355,40 +355,40 @@ class TestCookiesMiddleware:
headers={"Cookie": "a=b; c=d"},
cookies={"a": "new", "e": "f"},
)
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f")
def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request("http://example.org", cookies={"a": "á".encode()})
assert self.mw.process_request(req1, self.spider) is None
assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
# 2) Non UTF8-encoded bytes
req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")})
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
# 3) String
req3 = Request("http://example.org", cookies={"a": "á"})
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})
assert self.mw.process_request(req1, self.spider) is None
assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
# 2) Non UTF8-encoded bytes
req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")})
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
# 3) String
req3 = Request("http://example.org", headers={"Cookie": "a=á"})
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
def test_invalid_cookies(self):
@ -402,13 +402,13 @@ class TestCookiesMiddleware:
) as lc:
cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]
req1 = Request("http://example.org/1", cookies=cookies1)
assert self.mw.process_request(req1, self.spider) is None
assert self.mw.process_request(req1) is None
cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]
req2 = Request("http://example.org/2", cookies=cookies2)
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]
req3 = Request("http://example.org/3", cookies=cookies3)
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
lc.check(
(
"scrapy.downloadermiddlewares.cookies",
@ -436,22 +436,22 @@ class TestCookiesMiddleware:
def test_primitive_type_cookies(self):
# Boolean
req1 = Request("http://example.org", cookies={"a": True})
assert self.mw.process_request(req1, self.spider) is None
assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=True")
# Float
req2 = Request("http://example.org", cookies={"a": 9.5})
assert self.mw.process_request(req2, self.spider) is None
assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5")
# Integer
req3 = Request("http://example.org", cookies={"a": 10})
assert self.mw.process_request(req3, self.spider) is None
assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=10")
# String
req4 = Request("http://example.org", cookies={"a": "b"})
assert self.mw.process_request(req4, self.spider) is None
assert self.mw.process_request(req4) is None
self.assertCookieValEqual(req4.headers["Cookie"], b"a=b")
def _test_cookie_redirect(
@ -471,7 +471,7 @@ class TestCookiesMiddleware:
target.setdefault("status", 301)
request1 = Request(cookies=input_cookies, **source)
self.mw.process_request(request1, self.spider)
self.mw.process_request(request1)
cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None)
@ -481,16 +481,12 @@ class TestCookiesMiddleware:
},
**target,
)
assert self.mw.process_response(request1, response, self.spider) == response
assert self.mw.process_response(request1, response) == response
request2 = self.redirect_middleware.process_response(
request1,
response,
self.spider,
)
request2 = self.redirect_middleware.process_response(request1, response)
assert isinstance(request2, Request)
self.mw.process_request(request2, self.spider)
self.mw.process_request(request2)
cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None)
@ -564,11 +560,7 @@ class TestCookiesMiddleware:
**target,
)
request2 = self.redirect_middleware.process_response(
request1,
response,
self.spider,
)
request2 = self.redirect_middleware.process_response(request1, response)
assert isinstance(request2, Request)
cookies = request2.headers.get("Cookie")
@ -620,12 +612,12 @@ class TestCookiesMiddleware:
]
request1 = Request(url1, cookies=input_cookies)
self.mw.process_request(request1, self.spider)
self.mw.process_request(request1)
cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None)
request2 = Request(url2)
self.mw.process_request(request2, self.spider)
self.mw.process_request(request2)
cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None)
@ -674,7 +666,7 @@ class TestCookiesMiddleware:
cookies,
):
request1 = Request(url1)
self.mw.process_request(request1, self.spider)
self.mw.process_request(request1)
input_cookies = [
{
@ -688,10 +680,10 @@ class TestCookiesMiddleware:
"Set-Cookie": _cookies_to_set_cookie_list(input_cookies),
}
response = Response(url1, status=200, headers=headers)
assert self.mw.process_response(request1, response, self.spider) == response
assert self.mw.process_response(request1, response) == response
request2 = Request(url2)
self.mw.process_request(request2, self.spider)
self.mw.process_request(request2)
actual_cookies = request2.headers.get("Cookie")
assert actual_cookies == (b"a=b" if cookies else None)
@ -743,7 +735,7 @@ class TestCookiesMiddleware:
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
self.mw.process_request(request1, self.spider)
self.mw.process_request(request1)
cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None)
@ -752,18 +744,14 @@ class TestCookiesMiddleware:
headers={"Location": f"{to_scheme}://a.example"},
status=301,
)
assert self.mw.process_response(request1, response, self.spider) == response
assert self.mw.process_response(request1, response) == response
request2 = self.redirect_middleware.process_response(
request1,
response,
self.spider,
)
request2 = self.redirect_middleware.process_response(request1, response)
assert isinstance(request2, Request)
cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None)
self.mw.process_request(request2, self.spider)
self.mw.process_request(request2)
cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies3 else None)

13
tests/test_downloadermiddleware_defaultheaders.py
View File

@ -6,28 +6,27 @@ from scrapy.utils.test import get_crawler
class TestDefaultHeadersMiddleware:
def get_defaults_spider_mw(self):
def get_defaults_mw(self):
crawler = get_crawler(Spider)
spider = crawler._create_spider("foo")
defaults = {
to_bytes(k): [to_bytes(v)]
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
}
return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
def test_process_request(self):
defaults, spider, mw = self.get_defaults_spider_mw()
defaults, mw = self.get_defaults_mw()
req = Request("http://www.scrapytest.org")
mw.process_request(req, spider)
mw.process_request(req)
assert req.headers == defaults
def test_update_headers(self):
defaults, spider, mw = self.get_defaults_spider_mw()
defaults, mw = self.get_defaults_mw()
headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
req = Request("http://www.scrapytest.org", headers=headers)
assert req.headers == bytes_headers
mw.process_request(req, spider)
mw.process_request(req)
defaults.update(bytes_headers)
assert req.headers == defaults

8
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -14,20 +14,20 @@ class TestDownloadTimeoutMiddleware:
def test_default_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 180
def test_string_download_timeout(self):
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 20.1
def test_spider_has_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
spider.download_timeout = 2
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 2
def test_request_has_download_timeout(self):
@ -35,5 +35,5 @@ class TestDownloadTimeoutMiddleware:
spider.download_timeout = 2
mw.spider_opened(spider)
req.meta["download_timeout"] = 1
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 1

20
tests/test_downloadermiddleware_httpauth.py
View File

@ -36,48 +36,48 @@ class TestHttpAuthMiddlewareLegacy:
class TestHttpAuthMiddleware:
def setup_method(self):
self.mw = HttpAuthMiddleware()
self.spider = DomainSpider("foo")
self.mw.spider_opened(self.spider)
spider = DomainSpider("foo")
self.mw.spider_opened(spider)
def teardown_method(self):
del self.mw
def test_no_auth(self):
req = Request("http://example-noauth.com/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert "Authorization" not in req.headers
def test_auth_domain(self):
req = Request("http://example.com/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_subdomain(self):
req = Request("http://foo.example.com/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_already_set(self):
req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == b"Digest 123"
class TestHttpAuthAnyMiddleware:
def setup_method(self):
self.mw = HttpAuthMiddleware()
self.spider = AnyDomainSpider("foo")
self.mw.spider_opened(self.spider)
spider = AnyDomainSpider("foo")
self.mw.spider_opened(spider)
def teardown_method(self):
del self.mw
def test_auth(self):
req = Request("http://example.com/")
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_already_set(self):
req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == b"Digest 123"

184
tests/test_downloadermiddleware_httpcache.py
View File

@ -1,28 +1,36 @@
from __future__ import annotations
import email.utils
import shutil
import tempfile
import time
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import HtmlResponse, Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Generator
from scrapy.crawler import Crawler
class TestBase:
"""Base class with common setup and helper methods."""
policy_class: str
storage_class: str
def setup_method(self):
self.yesterday = email.utils.formatdate(time.time() - 86400)
self.today = email.utils.formatdate()
self.tomorrow = email.utils.formatdate(time.time() + 86400)
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("example.com")
self.tmpdir = tempfile.mkdtemp()
self.request = Request("http://www.example.com", headers={"User-Agent": "test"})
self.response = Response(
@ -31,13 +39,11 @@ class TestBase:
body=b"test body",
status=202,
)
self.crawler.stats.open_spider()
def teardown_method(self):
self.crawler.stats.close_spider()
shutil.rmtree(self.tmpdir)
def _get_settings(self, **new_settings):
def _get_settings(self, **new_settings: Any) -> dict[str, Any]:
settings = {
"HTTPCACHE_ENABLED": True,
"HTTPCACHE_DIR": self.tmpdir,
@ -47,27 +53,35 @@ class TestBase:
"HTTPCACHE_STORAGE": self.storage_class,
}
settings.update(new_settings)
return Settings(settings)
return settings
@contextmanager
def _storage(self, **new_settings):
with self._middleware(**new_settings) as mw:
yield mw.storage
@contextmanager
def _policy(self, **new_settings):
with self._middleware(**new_settings) as mw:
yield mw.policy
@contextmanager
def _middleware(self, **new_settings):
def _get_crawler(self, **new_settings: Any) -> Generator[Crawler]:
settings = self._get_settings(**new_settings)
mw = HttpCacheMiddleware(settings, self.crawler.stats)
mw.spider_opened(self.spider)
crawler = get_crawler(Spider, settings)
crawler.spider = crawler._create_spider("example.com")
assert crawler.stats
crawler.stats.open_spider()
try:
yield mw
yield crawler
finally:
mw.spider_closed(self.spider)
crawler.stats.close_spider()
@contextmanager
def _storage(self, **new_settings: Any):
with self._middleware(**new_settings) as mw:
yield mw.storage, mw.crawler
@contextmanager
def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]:
with self._get_crawler(**new_settings) as crawler:
assert crawler.spider
mw = HttpCacheMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
try:
yield mw
finally:
mw.spider_closed(crawler.spider)
def assertEqualResponse(self, response1, response2):
assert response1.url == response2.url
@ -94,37 +108,37 @@ class StorageTestMixin:
"""Mixin containing storage-specific test methods."""
def test_storage(self):
with self._storage() as storage:
with self._storage() as (storage, crawler):
request2 = self.request.copy()
assert storage.retrieve_response(self.spider, request2) is None
assert storage.retrieve_response(crawler.spider, request2) is None
storage.store_response(self.spider, self.request, self.response)
response2 = storage.retrieve_response(self.spider, request2)
storage.store_response(crawler.spider, self.request, self.response)
response2 = storage.retrieve_response(crawler.spider, request2)
assert isinstance(response2, HtmlResponse) # content-type header
self.assertEqualResponse(self.response, response2)
time.sleep(2) # wait for cache to expire
assert storage.retrieve_response(self.spider, request2) is None
assert storage.retrieve_response(crawler.spider, request2) is None
def test_storage_never_expire(self):
with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:
assert storage.retrieve_response(self.spider, self.request) is None
storage.store_response(self.spider, self.request, self.response)
with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler):
assert storage.retrieve_response(crawler.spider, self.request) is None
storage.store_response(crawler.spider, self.request, self.response)
time.sleep(0.5) # give the chance to expire
assert storage.retrieve_response(self.spider, self.request)
assert storage.retrieve_response(crawler.spider, self.request)
def test_storage_no_content_type_header(self):
"""Test that the response body is used to get the right response class
even if there is no Content-Type header"""
with self._storage() as storage:
assert storage.retrieve_response(self.spider, self.request) is None
with self._storage() as (storage, crawler):
assert storage.retrieve_response(crawler.spider, self.request) is None
response = Response(
"http://www.example.com",
body=b"<!DOCTYPE html>\n<title>.</title>",
status=202,
)
storage.store_response(self.spider, self.request, response)
cached_response = storage.retrieve_response(self.spider, self.request)
storage.store_response(crawler.spider, self.request, response)
cached_response = storage.retrieve_response(crawler.spider, self.request)
assert isinstance(cached_response, HtmlResponse)
self.assertEqualResponse(response, cached_response)
@ -135,15 +149,15 @@ class PolicyTestMixin:
def test_dont_cache(self):
with self._middleware() as mw:
self.request.meta["dont_cache"] = True
mw.process_response(self.request, self.response, self.spider)
assert mw.storage.retrieve_response(self.spider, self.request) is None
mw.process_response(self.request, self.response)
assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None
with self._middleware() as mw:
self.request.meta["dont_cache"] = False
mw.process_response(self.request, self.response, self.spider)
mw.process_response(self.request, self.response)
if mw.policy.should_cache_response(self.response, self.request):
assert isinstance(
mw.storage.retrieve_response(self.spider, self.request),
mw.storage.retrieve_response(mw.crawler.spider, self.request),
self.response.__class__,
)
@ -153,9 +167,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
def test_middleware(self):
with self._middleware() as mw:
assert mw.process_request(self.request, self.spider) is None
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response)
response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
@ -164,9 +178,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
with self._middleware() as mw:
req = Request("http://host.com/path")
res = Response("http://host2.net/test.html")
assert mw.process_request(req, self.spider) is None
mw.process_response(req, res, self.spider)
cached = mw.process_request(req, self.spider)
assert mw.process_request(req) is None
mw.process_response(req, res)
cached = mw.process_request(req)
assert isinstance(cached, Response)
self.assertEqualResponse(res, cached)
assert "cached" in cached.flags
@ -174,9 +188,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
def test_middleware_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
with pytest.raises(IgnoreRequest):
mw.process_request(self.request, self.spider)
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
mw.process_request(self.request)
mw.process_response(self.request, self.response)
response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
@ -185,10 +199,10 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# http responses are cached by default
req, res = Request("http://test.com/"), Response("http://test.com/")
with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None
mw.process_response(req, res, self.spider)
assert mw.process_request(req) is None
mw.process_response(req, res)
cached = mw.process_request(req, self.spider)
cached = mw.process_request(req)
assert isinstance(cached, Response), type(cached)
self.assertEqualResponse(res, cached)
assert "cached" in cached.flags
@ -196,19 +210,19 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# file response is not cached by default
req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt")
with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None
mw.process_response(req, res, self.spider)
assert mw.process_request(req) is None
mw.process_response(req, res)
assert mw.storage.retrieve_response(self.spider, req) is None
assert mw.process_request(req, self.spider) is None
assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
assert mw.process_request(req) is None
# s3 scheme response is cached by default
req, res = Request("s3://bucket/key"), Response("http://bucket/key")
with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None
mw.process_response(req, res, self.spider)
assert mw.process_request(req) is None
mw.process_response(req, res)
cached = mw.process_request(req, self.spider)
cached = mw.process_request(req)
assert isinstance(cached, Response), type(cached)
self.assertEqualResponse(res, cached)
assert "cached" in cached.flags
@ -216,25 +230,25 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# ignore s3 scheme
req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
assert mw.process_request(req, self.spider) is None
mw.process_response(req, res, self.spider)
assert mw.process_request(req) is None
mw.process_response(req, res)
assert mw.storage.retrieve_response(self.spider, req) is None
assert mw.process_request(req, self.spider) is None
assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
assert mw.process_request(req) is None
def test_middleware_ignore_http_codes(self):
# test response is not cached
with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw:
assert mw.process_request(self.request, self.spider) is None
mw.process_response(self.request, self.response, self.spider)
assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response)
assert mw.storage.retrieve_response(self.spider, self.request) is None
assert mw.process_request(self.request, self.spider) is None
assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None
assert mw.process_request(self.request) is None
# test response is cached
with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw:
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
mw.process_response(self.request, self.response)
response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
@ -243,14 +257,18 @@ class DummyPolicyTestMixin(PolicyTestMixin):
class RFC2616PolicyTestMixin(PolicyTestMixin):
"""Mixin containing RFC2616 policy specific test methods."""
def _process_requestresponse(self, mw, request, response):
@staticmethod
def _process_requestresponse(
mw: HttpCacheMiddleware, request: Request, response: Response | None
) -> Response | Request:
result = None
try:
result = mw.process_request(request, self.spider)
result = mw.process_request(request)
if result:
assert isinstance(result, (Request, Response))
return result
result = mw.process_response(request, response, self.spider)
assert response is not None
result = mw.process_response(request, response)
assert isinstance(result, Response)
return result
except Exception:
@ -270,11 +288,11 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# response for a request with no-store must not be cached
res1 = self._process_requestresponse(mw, req1, res0)
self.assertEqualResponse(res1, res0)
assert mw.storage.retrieve_response(self.spider, req1) is None
assert mw.storage.retrieve_response(mw.crawler.spider, req1) is None
# Re-do request without no-store and expect it to be cached
res2 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res2.flags
res3 = mw.process_request(req0, self.spider)
res3 = mw.process_request(req0)
assert "cached" in res3.flags
self.assertEqualResponse(res2, res3)
# request with no-cache directive must not return cached response
@ -330,7 +348,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
)
self.assertEqualResponse(res1, res0)
self.assertEqualResponse(res2, res0)
resc = mw.storage.retrieve_response(self.spider, req0)
resc = mw.storage.retrieve_response(mw.crawler.spider, req0)
if shouldcache:
self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags
@ -354,7 +372,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
)
self.assertEqualResponse(res1, res0)
self.assertEqualResponse(res2, res0)
resc = mw.storage.retrieve_response(self.spider, req0)
resc = mw.storage.retrieve_response(mw.crawler.spider, req0)
if shouldcache:
self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags
@ -421,7 +439,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# validate cached response if request max-age set as 0
req1 = req0.replace(headers={"Cache-Control": "max-age=0"})
res304 = res0.replace(status=304)
assert mw.process_request(req1, self.spider) is None
assert mw.process_request(req1) is None
res3 = self._process_requestresponse(mw, req1, res304)
self.assertEqualResponse(res1, res3)
assert "cached" in res3.flags
@ -513,14 +531,14 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
self._process_requestresponse(mw, req0, res0)
for e in mw.DOWNLOAD_EXCEPTIONS:
# Simulate encountering an error on download attempts
assert mw.process_request(req0, self.spider) is None
res1 = mw.process_exception(req0, e("foo"), self.spider)
assert mw.process_request(req0) is None
res1 = mw.process_exception(req0, e("foo"))
# Use cached response as recovery
assert "cached" in res1.flags
self.assertEqualResponse(res0, res1)
# Do not use cached response for unhandled exceptions
mw.process_request(req0, self.spider)
assert mw.process_exception(req0, Exception("foo"), self.spider) is None
mw.process_request(req0)
assert mw.process_exception(req0, Exception("foo")) is None
def test_ignore_response_cache_controls(self):
sampledata = [
@ -578,17 +596,17 @@ class TestDbmStorageWithRFC2616Policy(
class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy):
dbm_module = "tests.mocks.dummydbm"
def _get_settings(self, **new_settings):
def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)
return super()._get_settings(**new_settings)
def test_custom_dbm_module_loaded(self):
# make sure our dbm module has been loaded
with self._storage() as storage:
with self._storage() as (storage, _):
assert storage.dbmodule.__name__ == self.dbm_module
class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy):
def _get_settings(self, **new_settings):
def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_GZIP", True)
return super()._get_settings(**new_settings)

61
tests/test_downloadermiddleware_httpcompression.py
View File

@ -70,7 +70,6 @@ def _skip_if_no_zstd() -> None:
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider()
@ -124,7 +123,7 @@ class TestHttpCompression:
def test_process_request(self):
request = Request("http://scrapytest.org")
assert "Accept-Encoding" not in request.headers
self.mw.process_request(request, self.spider)
self.mw.process_request(request)
assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)
def test_process_response_gzip(self):
@ -132,7 +131,7 @@ class TestHttpCompression:
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -145,7 +144,7 @@ class TestHttpCompression:
response = self._getresponse("br")
request = response.request
assert response.headers["Content-Encoding"] == b"br"
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -172,7 +171,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
@ -197,7 +196,7 @@ class TestHttpCompression:
response = self._getresponse(check_key)
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
if raw_content is None:
raw_content = newresponse.body
else:
@ -221,7 +220,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
@ -241,7 +240,7 @@ class TestHttpCompression:
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -253,7 +252,7 @@ class TestHttpCompression:
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -265,7 +264,7 @@ class TestHttpCompression:
request = Request("http://scrapytest.org")
assert not response.headers.get("Content-Encoding")
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is response
assert newresponse.body.startswith(b"<!DOCTYPE")
self.assertStatsEqual("httpcompression/response_count", None)
@ -275,14 +274,14 @@ class TestHttpCompression:
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]
def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@ -296,7 +295,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
@ -314,7 +313,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@ -323,7 +322,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@ -331,7 +330,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@ -340,7 +339,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@ -362,7 +361,7 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
@ -387,7 +386,7 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
@ -410,7 +409,7 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/index")
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
@ -422,7 +421,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "application/gzip"
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -434,7 +433,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "application/octet-stream"
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -446,7 +445,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "binary/octet-stream"
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@ -494,7 +493,7 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert gunzip(newresponse.body) == plainbody
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 230)
@ -505,7 +504,7 @@ class TestHttpCompression:
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
newresponse = self.mw.process_response(request, response, self.spider)
newresponse = self.mw.process_response(request, response)
assert newresponse is response
assert response.body == b""
self.assertStatsEqual("httpcompression/response_count", None)
@ -520,7 +519,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
def test_compression_bomb_setting_br(self):
_skip_if_no_br()
@ -549,7 +548,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
@ -580,7 +579,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 10_000_000
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
def test_compression_bomb_request_meta_br(self):
_skip_if_no_br()
@ -611,7 +610,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
@ -655,7 +654,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
@ -701,7 +700,7 @@ class TestHttpCompression:
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
mw.process_response(response.request, response)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",

100
tests/test_downloadermiddleware_httpproxy.py
View File

@ -8,8 +8,6 @@ from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
spider = Spider("foo")
class TestHttpProxyMiddleware:
failureException = AssertionError # type: ignore[assignment]
@ -31,7 +29,7 @@ class TestHttpProxyMiddleware:
for url in ("http://e.com", "https://e.com", "file:///tmp/a"):
req = Request(url)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.url == url
assert req.meta == {}
@ -47,7 +45,7 @@ class TestHttpProxyMiddleware:
("file://tmp/a", None),
]:
req = Request(url)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.url == url
assert req.meta.get("proxy") == proxy
@ -55,14 +53,14 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://proxy.com"
mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"})
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta == {"proxy": "https://new.proxy:3128"}
def test_proxy_auth(self):
os.environ["http_proxy"] = "https://user:pass@proxy:3128"
mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjpwYXNz"
# proxy from request.meta
@ -70,7 +68,7 @@ class TestHttpProxyMiddleware:
"http://scrapytest.org",
meta={"proxy": "https://username:password@proxy:3128"},
)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert (
req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ="
@ -80,14 +78,14 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://user:@proxy:3128"
mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjo="
# proxy from request.meta
req = Request(
"http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"}
)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6"
@ -96,7 +94,7 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128"
mw = HttpProxyMiddleware(auth_encoding="utf-8")
req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic bcOhbjpwYXNz"
@ -104,14 +102,14 @@ class TestHttpProxyMiddleware:
req = Request(
"http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"}
)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic w7xzZXI6cGFzcw=="
# default latin-1 encoding
mw = HttpProxyMiddleware(auth_encoding="latin-1")
req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic beFuOnBhc3M="
@ -119,7 +117,7 @@ class TestHttpProxyMiddleware:
req = Request(
"http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"}
)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz"
@ -127,7 +125,7 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://proxy.for.http:3128"
mw = HttpProxyMiddleware()
req = Request("http://noproxy.com", meta={"proxy": None})
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert "proxy" in req.meta
assert req.meta["proxy"] is None
@ -137,23 +135,23 @@ class TestHttpProxyMiddleware:
os.environ["no_proxy"] = "*"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert "proxy" not in req.meta
os.environ["no_proxy"] = "other.com"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert "proxy" in req.meta
os.environ["no_proxy"] = "other.com,noproxy.com"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert "proxy" not in req.meta
# proxy from meta['proxy'] takes precedence
os.environ["no_proxy"] = "*"
req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"})
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.meta == {"proxy": "http://proxy.com"}
def test_no_proxy_invalid_values(self):
@ -166,18 +164,18 @@ class TestHttpProxyMiddleware:
def test_add_proxy_without_credentials(self):
middleware = HttpProxyMiddleware()
request = Request("https://example.com")
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers
def test_add_proxy_with_credentials(self):
middleware = HttpProxyMiddleware()
request = Request("https://example.com")
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header(
"user1",
@ -191,9 +189,9 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = None
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers
@ -203,9 +201,9 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = None
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers
@ -218,10 +216,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header(
"user1",
@ -237,9 +235,9 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user2:password2@example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header(
"user2",
@ -260,10 +258,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header(
"user1",
@ -273,7 +271,7 @@ class TestHttpProxyMiddleware:
request.meta["proxy"] = "https://example.com"
del request.headers[b"Proxy-Authorization"]
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers
@ -283,10 +281,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.org"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header(
"user1",
@ -300,10 +298,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.org"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header(
"user1",
@ -313,7 +311,7 @@ class TestHttpProxyMiddleware:
# Make sure, indirectly, that _auth_proxy is updated.
request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers
@ -323,10 +321,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user2:password2@example.org"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header(
"user2",
@ -342,9 +340,9 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.org"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta == {"proxy": "https://example.org"}
assert b"Proxy-Authorization" not in request.headers
@ -360,10 +358,10 @@ class TestHttpProxyMiddleware:
"https://example.com",
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.org"
del request.headers[b"Proxy-Authorization"]
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta == {"proxy": "https://example.org"}
assert b"Proxy-Authorization" not in request.headers
@ -373,7 +371,7 @@ class TestHttpProxyMiddleware:
"https://example.com",
headers={"Proxy-Authorization": "Basic foo"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert "proxy" not in request.meta
assert b"Proxy-Authorization" not in request.headers
@ -384,7 +382,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": "Basic foo"},
meta={"proxy": None},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers
@ -398,16 +396,16 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": "Basic foo"},
meta={"proxy": "https://example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic foo"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic foo"
request.headers["Proxy-Authorization"] = b"Basic bar"
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic bar"
@ -422,7 +420,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": b"Basic " + encoded_credentials},
meta={"proxy": "https://user1:password1@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials
@ -437,7 +435,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": b"Basic " + encoded_credentials1},
meta={"proxy": "https://user2:password2@example.com"},
)
assert middleware.process_request(request, spider) is None
assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com"
encoded_credentials2 = middleware._basic_auth_header(
"user2",

68
tests/test_downloadermiddleware_offsite.py
View File

@ -28,15 +28,15 @@ UNSET = object()
)
def test_process_request_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
assert mw.process_request(request, spider) is None
assert mw.process_request(request) is None
else:
with pytest.raises(IgnoreRequest):
mw.process_request(request, spider)
mw.process_request(request)
@pytest.mark.parametrize(
@ -50,18 +50,18 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
)
def test_process_request_dont_filter(value, filtered):
crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
kwargs = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
if filtered:
with pytest.raises(IgnoreRequest):
mw.process_request(request, spider)
mw.process_request(request)
else:
assert mw.process_request(request, spider) is None
assert mw.process_request(request) is None
@pytest.mark.parametrize(
@ -79,9 +79,9 @@ def test_process_request_dont_filter(value, filtered):
)
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
kwargs = {"meta": {}}
if allow_offsite is not UNSET:
kwargs["meta"]["allow_offsite"] = allow_offsite
@ -90,9 +90,9 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
request = Request("https://b.example", **kwargs)
if filtered:
with pytest.raises(IgnoreRequest):
mw.process_request(request, spider)
mw.process_request(request)
else:
assert mw.process_request(request, spider) is None
assert mw.process_request(request) is None
@pytest.mark.parametrize(
@ -108,27 +108,27 @@ def test_process_request_no_allowed_domains(value):
kwargs = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
spider = crawler._create_spider(name="a", **kwargs)
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.process_request(request, spider) is None
assert mw.process_request(request) is None
def test_process_request_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.process_request(request, spider) is None
assert mw.process_request(request) is None
for letter in ("b", "c"):
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
mw.process_request(request, spider)
mw.process_request(request)
@pytest.mark.parametrize(
@ -149,15 +149,15 @@ def test_process_request_invalid_domains():
)
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
assert mw.request_scheduled(request, spider) is None
assert mw.request_scheduled(request, crawler.spider) is None
else:
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider)
mw.request_scheduled(request, crawler.spider)
@pytest.mark.parametrize(
@ -171,18 +171,18 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
)
def test_request_scheduled_dont_filter(value, filtered):
crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
kwargs = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
if filtered:
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider)
mw.request_scheduled(request, crawler.spider)
else:
assert mw.request_scheduled(request, spider) is None
assert mw.request_scheduled(request, crawler.spider) is None
@pytest.mark.parametrize(
@ -198,24 +198,24 @@ def test_request_scheduled_no_allowed_domains(value):
kwargs = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
spider = crawler._create_spider(name="a", **kwargs)
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.request_scheduled(request, spider) is None
assert mw.request_scheduled(request, crawler.spider) is None
def test_request_scheduled_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.request_scheduled(request, spider) is None
assert mw.request_scheduled(request, crawler.spider) is None
for letter in ("b", "c"):
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider)
mw.request_scheduled(request, crawler.spider)

283
tests/test_downloadermiddleware_redirect.py
View File

File diff suppressed because it is too large Load Diff

65
tests/test_downloadermiddleware_retry.py
View File

@ -19,20 +19,21 @@ from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.settings.default_settings import RETRY_EXCEPTIONS
from scrapy.spiders import Spider
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
class TestRetry:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("foo")
self.crawler = get_crawler(DefaultSpider)
self.crawler.spider = self.crawler._create_spider()
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw.max_retry_times = 2
def test_priority_adjust(self):
req = Request("http://www.scrapytest.org/503")
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
req2 = self.mw.process_response(req, rsp, self.spider)
req2 = self.mw.process_response(req, rsp)
assert req2.priority < req.priority
def test_404(self):
@ -40,14 +41,14 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/404", body=b"", status=404)
# dont retry 404s
assert self.mw.process_response(req, rsp, self.spider) is rsp
assert self.mw.process_response(req, rsp) is rsp
def test_dont_retry(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
r = self.mw.process_response(req, rsp, self.spider)
r = self.mw.process_response(req, rsp)
assert r is rsp
# Test retry when dont_retry set to False
@ -55,13 +56,13 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503")
# first retry
r = self.mw.process_response(req, rsp, self.spider)
r = self.mw.process_response(req, rsp)
assert r is rsp
def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
r = self.mw.process_exception(req, DNSLookupError(), self.spider)
r = self.mw.process_exception(req, DNSLookupError())
assert r is None
def test_503(self):
@ -69,17 +70,17 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp, self.spider)
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
# second retry
req = self.mw.process_response(req, rsp, self.spider)
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
# discard it
assert self.mw.process_response(req, rsp, self.spider) is rsp
assert self.mw.process_response(req, rsp) is rsp
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
@ -118,7 +119,8 @@ class TestRetry:
settings_dict = {
"RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc],
}
crawler = get_crawler(Spider, settings_dict=settings_dict)
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
@ -128,65 +130,61 @@ class TestRetry:
mw = self.mw
# first retry
req = mw.process_exception(req, exception, self.spider)
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
# second retry
req = mw.process_exception(req, exception, self.spider)
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
# discard it
req = mw.process_exception(req, exception, self.spider)
req = mw.process_exception(req, exception)
assert req is None
class TestMaxRetryTimes:
invalid_url = "http://www.scrapytest.org/invalid_url"
def get_spider_and_middleware(self, settings=None):
crawler = get_crawler(Spider, settings or {})
spider = crawler._create_spider("foo")
middleware = RetryMiddleware.from_crawler(crawler)
return spider, middleware
def get_middleware(self, settings=None):
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
def test_with_settings_zero(self):
max_retry_times = 0
settings = {"RETRY_TIMES": max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
middleware = self.get_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(
req,
DNSLookupError("foo"),
max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_metakey_zero(self):
max_retry_times = 0
spider, middleware = self.get_spider_and_middleware()
middleware = self.get_middleware()
meta = {"max_retry_times": max_retry_times}
req = Request(self.invalid_url, meta=meta)
self._test_retry(
req,
DNSLookupError("foo"),
max_retry_times,
spider=spider,
middleware=middleware,
)
def test_without_metakey(self):
max_retry_times = 5
settings = {"RETRY_TIMES": max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
middleware = self.get_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(
req,
DNSLookupError("foo"),
max_retry_times,
spider=spider,
middleware=middleware,
)
@ -198,20 +196,18 @@ class TestMaxRetryTimes:
req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
middleware = self.get_middleware(settings)
self._test_retry(
req1,
DNSLookupError("foo"),
meta_max_retry_times,
spider=spider,
middleware=middleware,
)
self._test_retry(
req2,
DNSLookupError("foo"),
middleware_max_retry_times,
spider=spider,
middleware=middleware,
)
@ -223,26 +219,24 @@ class TestMaxRetryTimes:
req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
middleware = self.get_middleware(settings)
self._test_retry(
req1,
DNSLookupError("foo"),
meta_max_retry_times,
spider=spider,
middleware=middleware,
)
self._test_retry(
req2,
DNSLookupError("foo"),
middleware_max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_dont_retry(self):
max_retry_times = 4
spider, middleware = self.get_spider_and_middleware()
middleware = self.get_middleware()
meta = {
"max_retry_times": max_retry_times,
"dont_retry": True,
@ -252,7 +246,6 @@ class TestMaxRetryTimes:
req,
DNSLookupError("foo"),
0,
spider=spider,
middleware=middleware,
)
@ -261,18 +254,16 @@ class TestMaxRetryTimes:
req,
exception,
max_retry_times,
spider=None,
middleware=None,
):
spider = spider or self.spider
middleware = middleware or self.mw
for i in range(max_retry_times):
req = middleware.process_exception(req, exception, spider)
req = middleware.process_exception(req, exception)
assert isinstance(req, Request)
# discard it
req = middleware.process_exception(req, exception, spider)
req = middleware.process_exception(req, exception)
assert req is None

43
tests/test_downloadermiddleware_robotstxt.py
View File

@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
from scrapy.http.request import NO_CALLBACK
@ -83,10 +82,10 @@ Disallow: /some/randome/page.html
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
d1 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type]
middleware.process_request(Request("http://site.local/allowed1"))
)
d2 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type]
middleware.process_request(Request("http://site.local/allowed2"))
)
await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True))
@ -94,8 +93,8 @@ Disallow: /some/randome/page.html
@deferred_f_from_coro_f
async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
c1 = middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type]
c2 = middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type]
c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2)
@deferred_f_from_coro_f
@ -164,7 +163,7 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@deferred_f_from_coro_f
async def test_robotstxt_error(self):
async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None:
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = error.DNSLookupError("Robotstxt address not found")
@ -176,9 +175,8 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
await middleware.process_request(Request("http://site.local"), None)
assert middleware._logerror.called
await middleware.process_request(Request("http://site.local"))
assert "DNS lookup failed: Robotstxt address not found" in caplog.text
@deferred_f_from_coro_f
async def test_robotstxt_immediate_error(self):
@ -205,10 +203,13 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = ignore_request
middleware = RobotsTxtMiddleware(self.crawler)
mw_module_logger.error = mock.MagicMock()
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
assert not mw_module_logger.error.called # type: ignore[attr-defined]
with mock.patch(
"scrapy.downloadermiddlewares.robotstxt.logger"
) as mw_module_logger:
await self.assertNotIgnored(
Request("http://site.local/allowed"), middleware
)
assert not mw_module_logger.error.called
def test_robotstxt_user_agent_setting(self):
crawler = self._get_successful_crawler()
@ -216,7 +217,7 @@ Disallow: /some/randome/page.html
crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")
middleware = RobotsTxtMiddleware(crawler)
rp = mock.MagicMock(return_value=True)
middleware.process_request_2(rp, Request("http://site.local/allowed"), None)
middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@deferred_f_from_coro_f
@ -224,34 +225,30 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock()
await middleware.process_request(
Request("data:text/plain,Hello World data"), None
)
await middleware.process_request(Request("data:text/plain,Hello World data"))
assert not middleware.process_request_2.called
await middleware.process_request(
Request("file:///tests/sample_data/test_site/nothinghere.html"), None
Request("file:///tests/sample_data/test_site/nothinghere.html")
)
assert not middleware.process_request_2.called
await middleware.process_request(Request("http://site.local/allowed"), None)
await middleware.process_request(Request("http://site.local/allowed"))
assert middleware.process_request_2.called
async def assertNotIgnored(
self, request: Request, middleware: RobotsTxtMiddleware
) -> None:
spider = None # not actually used
try:
await middleware.process_request(request, spider) # type: ignore[arg-type]
await middleware.process_request(request)
except IgnoreRequest:
pytest.fail("IgnoreRequest was raised unexpectedly")
async def assertIgnored(
self, request: Request, middleware: RobotsTxtMiddleware
) -> None:
spider = None # not actually used
with pytest.raises(IgnoreRequest):
await middleware.process_request(request, spider) # type: ignore[arg-type]
await middleware.process_request(request)
def assertRobotsTxtRequested(self, base_url: str) -> None:
calls = self.crawler.engine.download_async.call_args_list

7
tests/test_downloadermiddleware_stats.py
View File

@ -11,7 +11,6 @@ class MyException(Exception):
class TestDownloaderStats:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = DownloaderStats(self.crawler.stats)
self.crawler.stats.open_spider()
@ -25,15 +24,15 @@ class TestDownloaderStats:
)
def test_process_request(self):
self.mw.process_request(self.req, self.spider)
self.mw.process_request(self.req)
self.assertStatsEqual("downloader/request_count", 1)
def test_process_response(self):
self.mw.process_response(self.req, self.res, self.spider)
self.mw.process_response(self.req, self.res)
self.assertStatsEqual("downloader/response_count", 1)
def test_process_exception(self):
self.mw.process_exception(self.req, MyException(), self.spider)
self.mw.process_exception(self.req, MyException())
self.assertStatsEqual("downloader/exception_count", 1)
self.assertStatsEqual(
"downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException",

12
tests/test_downloadermiddleware_useragent.py
View File

@ -11,9 +11,9 @@ class TestUserAgentMiddleware:
return spider, UserAgentMiddleware.from_crawler(crawler)
def test_default_agent(self):
spider, mw = self.get_spider_and_mw("default_useragent")
_, mw = self.get_spider_and_mw("default_useragent")
req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"default_useragent"
def test_remove_agent(self):
@ -22,7 +22,7 @@ class TestUserAgentMiddleware:
spider.user_agent = None
mw.spider_opened(spider)
req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.headers.get("User-Agent") is None
def test_spider_agent(self):
@ -30,7 +30,7 @@ class TestUserAgentMiddleware:
spider.user_agent = "spider_useragent"
mw.spider_opened(spider)
req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"spider_useragent"
def test_header_agent(self):
@ -40,7 +40,7 @@ class TestUserAgentMiddleware:
req = Request(
"http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
)
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"header_useragent"
def test_no_agent(self):
@ -48,5 +48,5 @@ class TestUserAgentMiddleware:
spider.user_agent = None
mw.spider_opened(spider)
req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None
assert mw.process_request(req) is None
assert "User-Agent" not in req.headers

4
tests/test_middleware.py
View File

@ -20,7 +20,7 @@ class M1:
def close_spider(self, spider):
pass
def process(self, response, request, spider):
def process(self, response, request):
pass
@ -33,7 +33,7 @@ class M2:
class M3:
def process(self, response, request, spider):
def process(self, response, request):
pass

19
tests/test_request_attribute_binding.py
View File

@ -11,18 +11,18 @@ OVERRIDDEN_URL = "https://example.org"
class ProcessResponseMiddleware:
def process_response(self, request, response, spider):
def process_response(self, request, response):
return response.replace(request=Request(OVERRIDDEN_URL))
class RaiseExceptionRequestMiddleware:
def process_request(self, request, spider):
def process_request(self, request):
1 / 0
return request
class CatchExceptionOverrideRequestMiddleware:
def process_exception(self, request, exception, spider):
def process_exception(self, request, exception):
return Response(
url="http://localhost/",
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
@ -31,7 +31,7 @@ class CatchExceptionOverrideRequestMiddleware:
class CatchExceptionDoNotOverrideRequestMiddleware:
def process_exception(self, request, exception, spider):
def process_exception(self, request, exception):
return Response(
url="http://localhost/",
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
@ -46,10 +46,17 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
class AlternativeCallbacksMiddleware:
def process_response(self, request, response, spider):
def __init__(self, crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_response(self, request, response):
new_request = request.replace(
url=OVERRIDDEN_URL,
callback=spider.alt_callback,
callback=self.crawler.spider.alt_callback,
cb_kwargs={"foo": "bar"},
)
return response.replace(request=new_request)

4
tests/test_request_cb_kwargs.py
View File

@ -12,11 +12,11 @@ class InjectArgumentsDownloaderMiddleware:
Make sure downloader middlewares are able to update the keyword arguments
"""
def process_request(self, request, spider):
def process_request(self, request):
if request.callback.__name__ == "parse_downloader_mw":
request.cb_kwargs["from_process_request"] = True
def process_response(self, request, response, spider):
def process_response(self, request, response):
if request.callback.__name__ == "parse_downloader_mw":
request.cb_kwargs["from_process_response"] = True
return response

31
tests/test_spidermiddleware_referer.py
View File

@ -31,15 +31,13 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
)
from scrapy.spiders import Spider
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Sequence
@pytest.fixture
def spider() -> Spider:
return Spider("foo")
from scrapy.crawler import Crawler
class TestRefererMiddleware:
@ -1011,20 +1009,24 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
]
@pytest.fixture
def referrermw(self) -> RefererMiddleware:
settings = Settings(self.settings)
return RefererMiddleware(settings)
def crawler(self) -> Crawler:
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
return crawler
@pytest.fixture
def redirectmw(self) -> RedirectMiddleware:
settings = Settings(self.settings)
return RedirectMiddleware(settings)
def referrermw(self, crawler: Crawler) -> RefererMiddleware:
return RefererMiddleware.from_crawler(crawler)
@pytest.fixture
def redirectmw(self, crawler: Crawler) -> RedirectMiddleware:
return RedirectMiddleware.from_crawler(crawler)
def test( # type: ignore[override]
self,
crawler: Crawler,
referrermw: RefererMiddleware,
redirectmw: RedirectMiddleware,
spider: Spider,
) -> None:
for (
parent,
@ -1044,9 +1046,10 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
request.url, headers={"Location": url}, status=status
)
request = cast(
"Request", redirectmw.process_response(request, response, spider)
"Request", redirectmw.process_response(request, response)
)
referrermw.request_scheduled(request, spider)
assert crawler.spider
referrermw.request_scheduled(request, crawler.spider)
assert isinstance(request, Request)
assert request.headers.get("Referer") == final_referrer