RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate the spider arg of downloader middleware methods. (#7037) 

* Deprecate the spider arg of downloader middleware methods.

* Add a test for deprecated spider args in downloader mws.
This commit is contained in:
Andrey Rakhmatullin 2025-09-02 20:51:47 +04:00 committed by GitHub
parent c097921c44
commit 2ad81a0ef8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 728 additions and 665 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/topics/downloader-middleware.rst
View File

@ -70,7 +70,7 @@ defines one or more of these methods:
.. note:: Any of the downloader middleware methods may also return a deferred. .. note:: Any of the downloader middleware methods may also return a deferred.
.. method:: process_request(request, spider) .. method:: process_request(request)
This method is called for each request that goes through the download This method is called for each request that goes through the download
middleware. middleware.
@ -102,10 +102,7 @@ defines one or more of these methods:
:param request: the request being processed :param request: the request being processed
:type request: :class:`~scrapy.Request` object :type request: :class:`~scrapy.Request` object
:param spider: the spider for which this request is intended .. method:: process_response(request, response)
:type spider: :class:`~scrapy.Spider` object
.. method:: process_response(request, response, spider)
:meth:`process_response` should either: return a :class:`~scrapy.http.Response` :meth:`process_response` should either: return a :class:`~scrapy.http.Response`
object, return a :class:`~scrapy.Request` object or object, return a :class:`~scrapy.Request` object or
@ -129,10 +126,7 @@ defines one or more of these methods:
:param response: the response being processed :param response: the response being processed
:type response: :class:`~scrapy.http.Response` object :type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended .. method:: process_exception(request, exception)
:type spider: :class:`~scrapy.Spider` object
.. method:: process_exception(request, exception, spider)
Scrapy calls :meth:`process_exception` when a download handler Scrapy calls :meth:`process_exception` when a download handler
or a :meth:`process_request` (from a downloader middleware) raises an or a :meth:`process_request` (from a downloader middleware) raises an
@ -160,9 +154,6 @@ defines one or more of these methods:
:param exception: the raised exception :param exception: the raised exception
:type exception: an ``Exception`` object :type exception: an ``Exception`` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.Spider` object
.. _topics-downloader-middleware-ref: .. _topics-downloader-middleware-ref:
Built-in downloader middleware reference Built-in downloader middleware reference

36
scrapy/core/downloader/middleware.py
View File

@ -35,10 +35,13 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def _add_middleware(self, mw: Any) -> None: def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "process_request"): if hasattr(mw, "process_request"):
self.methods["process_request"].append(mw.process_request) self.methods["process_request"].append(mw.process_request)
self._check_mw_method_spider_arg(mw.process_request)
if hasattr(mw, "process_response"): if hasattr(mw, "process_response"):
self.methods["process_response"].appendleft(mw.process_response) self.methods["process_response"].appendleft(mw.process_response)
self._check_mw_method_spider_arg(mw.process_response)
if hasattr(mw, "process_exception"): if hasattr(mw, "process_exception"):
self.methods["process_exception"].appendleft(mw.process_exception) self.methods["process_exception"].appendleft(mw.process_exception)
self._check_mw_method_spider_arg(mw.process_exception)
@inlineCallbacks @inlineCallbacks
def download( def download(
@ -64,9 +67,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, Response | Request]: ) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_request"]: for method in self.methods["process_request"]:
method = cast("Callable", method) method = cast("Callable", method)
response = yield deferred_from_coro( if method in self._mw_methods_requiring_spider:
method(request=request, spider=self._spider) response = yield deferred_from_coro(
) method(request=request, spider=self._spider)
)
else:
response = yield deferred_from_coro(method(request=request))
if response is not None and not isinstance( if response is not None and not isinstance(
response, (Response, Request) response, (Response, Request)
): ):
@ -91,9 +97,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods["process_response"]: for method in self.methods["process_response"]:
method = cast("Callable", method) method = cast("Callable", method)
response = yield deferred_from_coro( if method in self._mw_methods_requiring_spider:
method(request=request, response=response, spider=self._spider) response = yield deferred_from_coro(
) method(request=request, response=response, spider=self._spider)
)
else:
response = yield deferred_from_coro(
method(request=request, response=response)
)
if not isinstance(response, (Response, Request)): if not isinstance(response, (Response, Request)):
raise _InvalidOutput( raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, " f"Middleware {method.__qualname__} must return Response or Request, "
@ -109,9 +120,16 @@ class DownloaderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, Response | Request]: ) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_exception"]: for method in self.methods["process_exception"]:
method = cast("Callable", method) method = cast("Callable", method)
response = yield deferred_from_coro( if method in self._mw_methods_requiring_spider:
method(request=request, exception=exception, spider=self._spider) response = yield deferred_from_coro(
) method(
request=request, exception=exception, spider=self._spider
)
)
else:
response = yield deferred_from_coro(
method(request=request, exception=exception)
)
if response is not None and not isinstance( if response is not None and not isinstance(
response, (Response, Request) response, (Response, Request)
): ):

25
scrapy/downloadermiddlewares/cookies.py
View File

@ -9,6 +9,7 @@ from tldextract import TLDExtract
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.http import Response from scrapy.http import Response
from scrapy.http.cookies import CookieJar from scrapy.http.cookies import CookieJar
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode from scrapy.utils.python import to_unicode
@ -39,6 +40,8 @@ def _is_public_domain(domain: str) -> bool:
class CookiesMiddleware: class CookiesMiddleware:
"""This middleware enables working with sites that need cookies""" """This middleware enables working with sites that need cookies"""
crawler: Crawler
def __init__(self, debug: bool = False): def __init__(self, debug: bool = False):
self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar)
self.debug: bool = debug self.debug: bool = debug
@ -47,7 +50,9 @@ class CookiesMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COOKIES_ENABLED"): if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured raise NotConfigured
return cls(crawler.settings.getbool("COOKIES_DEBUG")) o = cls(crawler.settings.getbool("COOKIES_DEBUG"))
o.crawler = crawler
return o
def _process_cookies( def _process_cookies(
self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
@ -67,8 +72,9 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request) jar.set_cookie_if_ok(cookie, request)
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
if request.meta.get("dont_merge_cookies", False): if request.meta.get("dont_merge_cookies", False):
return None return None
@ -81,11 +87,12 @@ class CookiesMiddleware:
# set Cookie header # set Cookie header
request.headers.pop("Cookie", None) request.headers.pop("Cookie", None)
jar.add_cookie_header(request) jar.add_cookie_header(request)
self._debug_cookie(request, spider) self._debug_cookie(request)
return None return None
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if request.meta.get("dont_merge_cookies", False): if request.meta.get("dont_merge_cookies", False):
return response return response
@ -96,11 +103,11 @@ class CookiesMiddleware:
cookies = jar.make_cookies(response, request) cookies = jar.make_cookies(response, request)
self._process_cookies(cookies, jar=jar, request=request) self._process_cookies(cookies, jar=jar, request=request)
self._debug_set_cookie(response, spider) self._debug_set_cookie(response)
return response return response
def _debug_cookie(self, request: Request, spider: Spider) -> None: def _debug_cookie(self, request: Request) -> None:
if self.debug: if self.debug:
cl = [ cl = [
to_unicode(c, errors="replace") to_unicode(c, errors="replace")
@ -109,9 +116,9 @@ class CookiesMiddleware:
if cl: if cl:
cookies = "\n".join(f"Cookie: {c}\n" for c in cl) cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = f"Sending cookies to: {request}\n{cookies}" msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={"spider": spider}) logger.debug(msg, extra={"spider": self.crawler.spider})
def _debug_set_cookie(self, response: Response, spider: Spider) -> None: def _debug_set_cookie(self, response: Response) -> None:
if self.debug: if self.debug:
cl = [ cl = [
to_unicode(c, errors="replace") to_unicode(c, errors="replace")
@ -120,7 +127,7 @@ class CookiesMiddleware:
if cl: if cl:
cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = f"Received cookies from: {response}\n{cookies}" msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider}) logger.debug(msg, extra={"spider": self.crawler.spider})
def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None: def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None:
""" """

4
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -8,6 +8,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import without_none_values from scrapy.utils.python import without_none_values
if TYPE_CHECKING: if TYPE_CHECKING:
@ -30,8 +31,9 @@ class DefaultHeadersMiddleware:
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items()) return cls(headers.items())
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
for k, v in self._headers: for k, v in self._headers:
request.headers.setdefault(k, v) request.headers.setdefault(k, v)

4
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -9,6 +9,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING: if TYPE_CHECKING:
# typing.Self requires Python 3.11 # typing.Self requires Python 3.11
@ -31,8 +32,9 @@ class DownloadTimeoutMiddleware:
def spider_opened(self, spider: Spider) -> None: def spider_opened(self, spider: Spider) -> None:
self._timeout = getattr(spider, "download_timeout", self._timeout) self._timeout = getattr(spider, "download_timeout", self._timeout)
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
if self._timeout: if self._timeout:
request.meta.setdefault("download_timeout", self._timeout) request.meta.setdefault("download_timeout", self._timeout)

4
scrapy/downloadermiddlewares/httpauth.py
View File

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from w3lib.http import basic_auth_header from w3lib.http import basic_auth_header
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.url import url_is_from_any_domain from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING: if TYPE_CHECKING:
@ -38,8 +39,9 @@ class HttpAuthMiddleware:
self.auth = basic_auth_header(usr, pwd) self.auth = basic_auth_header(usr, pwd)
self.domain = spider.http_auth_domain # type: ignore[attr-defined] self.domain = spider.http_auth_domain # type: ignore[attr-defined]
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
auth = getattr(self, "auth", None) auth = getattr(self, "auth", None)
if ( if (

29
scrapy/downloadermiddlewares/httpcache.py
View File

@ -17,6 +17,7 @@ from twisted.web.client import ResponseFailed
from scrapy import signals from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
if TYPE_CHECKING: if TYPE_CHECKING:
@ -45,6 +46,8 @@ class HttpCacheMiddleware:
OSError, OSError,
) )
crawler: Crawler
def __init__(self, settings: Settings, stats: StatsCollector) -> None: def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool("HTTPCACHE_ENABLED"): if not settings.getbool("HTTPCACHE_ENABLED"):
raise NotConfigured raise NotConfigured
@ -59,6 +62,7 @@ class HttpCacheMiddleware:
o = cls(crawler.settings, crawler.stats) o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
o.crawler = crawler
return o return o
def spider_opened(self, spider: Spider) -> None: def spider_opened(self, spider: Spider) -> None:
@ -67,8 +71,9 @@ class HttpCacheMiddleware:
def spider_closed(self, spider: Spider) -> None: def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider) self.storage.close_spider(spider)
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
if request.meta.get("dont_cache", False): if request.meta.get("dont_cache", False):
return None return None
@ -80,7 +85,7 @@ class HttpCacheMiddleware:
# Look for cached response and check if expired # Look for cached response and check if expired
cachedresponse: Response | None = self.storage.retrieve_response( cachedresponse: Response | None = self.storage.retrieve_response(
spider, request self.crawler.spider, request
) )
if cachedresponse is None: if cachedresponse is None:
self.stats.inc_value("httpcache/miss") self.stats.inc_value("httpcache/miss")
@ -101,8 +106,9 @@ class HttpCacheMiddleware:
return None return None
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if request.meta.get("dont_cache", False): if request.meta.get("dont_cache", False):
return response return response
@ -121,7 +127,7 @@ class HttpCacheMiddleware:
cachedresponse: Response | None = request.meta.pop("cached_response", None) cachedresponse: Response | None = request.meta.pop("cached_response", None)
if cachedresponse is None: if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand") self.stats.inc_value("httpcache/firsthand")
self._cache_response(spider, response, request, cachedresponse) self._cache_response(response, request)
return response return response
if self.policy.is_cached_response_valid(cachedresponse, response, request): if self.policy.is_cached_response_valid(cachedresponse, response, request):
@ -129,11 +135,12 @@ class HttpCacheMiddleware:
return cachedresponse return cachedresponse
self.stats.inc_value("httpcache/invalidate") self.stats.inc_value("httpcache/invalidate")
self._cache_response(spider, response, request, cachedresponse) self._cache_response(response, request)
return response return response
@_warn_spider_arg
def process_exception( def process_exception(
self, request: Request, exception: Exception, spider: Spider self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
cachedresponse: Response | None = request.meta.pop("cached_response", None) cachedresponse: Response | None = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance( if cachedresponse is not None and isinstance(
@ -143,15 +150,9 @@ class HttpCacheMiddleware:
return cachedresponse return cachedresponse
return None return None
def _cache_response( def _cache_response(self, response: Response, request: Request) -> None:
self,
spider: Spider,
response: Response,
request: Request,
cachedresponse: Response | None,
) -> None:
if self.policy.should_cache_response(response, request): if self.policy.should_cache_response(response, request):
self.stats.inc_value("httpcache/store") self.stats.inc_value("httpcache/store")
self.storage.store_response(spider, request, response) self.storage.store_response(self.crawler.spider, request, response)
else: else:
self.stats.inc_value("httpcache/uncacheable") self.stats.inc_value("httpcache/uncacheable")

7
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -15,6 +15,7 @@ from scrapy.utils._compression import (
_unbrotli, _unbrotli,
_unzstd, _unzstd,
) )
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.gz import gunzip from scrapy.utils.gz import gunzip
if TYPE_CHECKING: if TYPE_CHECKING:
@ -93,14 +94,16 @@ class HttpCompressionMiddleware:
) )
self._warn_size = spider.download_warnsize self._warn_size = spider.download_warnsize
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
return None return None
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if request.method == "HEAD": if request.method == "HEAD":
return response return response

4
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -10,6 +10,7 @@ from urllib.request import ( # type: ignore[attr-defined]
) )
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes from scrapy.utils.python import to_bytes
@ -55,8 +56,9 @@ class HttpProxyMiddleware:
return creds, proxy_url return creds, proxy_url
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
creds, proxy_url, scheme = None, None, None creds, proxy_url, scheme = None, None, None
if "proxy" in request.meta: if "proxy" in request.meta:

22
scrapy/downloadermiddlewares/offsite.py
View File

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.exceptions import IgnoreRequest from scrapy.exceptions import IgnoreRequest
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING: if TYPE_CHECKING:
@ -21,29 +22,34 @@ logger = logging.getLogger(__name__)
class OffsiteMiddleware: class OffsiteMiddleware:
crawler: Crawler
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen: set[str] = set()
@classmethod @classmethod
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats assert crawler.stats
o = cls(crawler.stats) o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
o.crawler = crawler
return o return o
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen: set[str] = set()
def spider_opened(self, spider: Spider) -> None: def spider_opened(self, spider: Spider) -> None:
self.host_regex: re.Pattern[str] = self.get_host_regex(spider) self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
def request_scheduled(self, request: Request, spider: Spider) -> None: def request_scheduled(self, request: Request, spider: Spider) -> None:
self.process_request(request, spider) self.process_request(request)
def process_request(self, request: Request, spider: Spider) -> None: @_warn_spider_arg
def process_request(self, request: Request, spider: Spider | None = None) -> None:
assert self.crawler.spider
if ( if (
request.dont_filter request.dont_filter
or request.meta.get("allow_offsite") or request.meta.get("allow_offsite")
or self.should_follow(request, spider) or self.should_follow(request, self.crawler.spider)
): ):
return return
domain = urlparse_cached(request).hostname domain = urlparse_cached(request).hostname
@ -52,7 +58,7 @@ class OffsiteMiddleware:
logger.debug( logger.debug(
"Filtered offsite request to %(domain)r: %(request)s", "Filtered offsite request to %(domain)r: %(request)s",
{"domain": domain, "request": request}, {"domain": domain, "request": request},
extra={"spider": spider}, extra={"spider": self.crawler.spider},
) )
self.stats.inc_value("offsite/domains") self.stats.inc_value("offsite/domains")
self.stats.inc_value("offsite/filtered") self.stats.inc_value("offsite/filtered")

29
scrapy/downloadermiddlewares/redirect.py
View File

@ -8,6 +8,7 @@ from w3lib.url import safe_url_string
from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Response from scrapy.http import HtmlResponse, Response
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh from scrapy.utils.response import get_meta_refresh
@ -79,6 +80,7 @@ def _build_redirect_request(
class BaseRedirectMiddleware: class BaseRedirectMiddleware:
crawler: Crawler
enabled_setting: str = "REDIRECT_ENABLED" enabled_setting: str = "REDIRECT_ENABLED"
def __init__(self, settings: BaseSettings): def __init__(self, settings: BaseSettings):
@ -90,11 +92,11 @@ class BaseRedirectMiddleware:
@classmethod @classmethod
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings) o = cls(crawler.settings)
o.crawler = crawler
return o
def _redirect( def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request:
self, redirected: Request, request: Request, spider: Spider, reason: Any
) -> Request:
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1 redirects = request.meta.get("redirect_times", 0) + 1
@ -114,13 +116,13 @@ class BaseRedirectMiddleware:
logger.debug( logger.debug(
"Redirecting (%(reason)s) to %(redirected)s from %(request)s", "Redirecting (%(reason)s) to %(redirected)s from %(request)s",
{"reason": reason, "redirected": redirected, "request": request}, {"reason": reason, "redirected": redirected, "request": request},
extra={"spider": spider}, extra={"spider": self.crawler.spider},
) )
return redirected return redirected
logger.debug( logger.debug(
"Discarding %(request)s: max redirections reached", "Discarding %(request)s: max redirections reached",
{"request": request}, {"request": request},
extra={"spider": spider}, extra={"spider": self.crawler.spider},
) )
raise IgnoreRequest("max redirections reached") raise IgnoreRequest("max redirections reached")
@ -144,12 +146,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
and meta-refresh html tag. and meta-refresh html tag.
""" """
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if ( if (
request.meta.get("dont_redirect", False) request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", []) or response.status
in getattr(self.crawler.spider, "handle_httpstatus_list", [])
or response.status in request.meta.get("handle_httpstatus_list", []) or response.status in request.meta.get("handle_httpstatus_list", [])
or request.meta.get("handle_httpstatus_all", False) or request.meta.get("handle_httpstatus_all", False)
): ):
@ -171,10 +175,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
return response return response
if response.status in (301, 307, 308) or request.method == "HEAD": if response.status in (301, 307, 308) or request.method == "HEAD":
return self._redirect(redirected, request, spider, response.status) return self._redirect(redirected, request, response.status)
redirected = self._redirect_request_using_get(request, redirected_url) redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status) return self._redirect(redirected, request, response.status)
class MetaRefreshMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware):
@ -185,8 +189,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if ( if (
request.meta.get("dont_redirect", False) request.meta.get("dont_redirect", False)
@ -203,5 +208,5 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
if urlparse_cached(redirected).scheme not in {"http", "https"}: if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response return response
if cast("float", interval) < self._maxdelay: if cast("float", interval) < self._maxdelay:
return self._redirect(redirected, request, spider, "meta refresh") return self._redirect(redirected, request, "meta refresh")
return response return response

25
scrapy/downloadermiddlewares/retry.py
View File

@ -16,6 +16,7 @@ from logging import Logger, getLogger
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message from scrapy.utils.response import response_status_message
@ -123,6 +124,8 @@ def get_retry_request(
class RetryMiddleware: class RetryMiddleware:
crawler: Crawler
def __init__(self, settings: BaseSettings): def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"): if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured raise NotConfigured
@ -136,39 +139,41 @@ class RetryMiddleware:
@classmethod @classmethod
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings) o = cls(crawler.settings)
o.crawler = crawler
return o
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
if request.meta.get("dont_retry", False): if request.meta.get("dont_retry", False):
return response return response
if response.status in self.retry_http_codes: if response.status in self.retry_http_codes:
reason = response_status_message(response.status) reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response return self._retry(request, reason) or response
return response return response
@_warn_spider_arg
def process_exception( def process_exception(
self, request: Request, exception: Exception, spider: Spider self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False "dont_retry", False
): ):
return self._retry(request, exception, spider) return self._retry(request, exception)
return None return None
def _retry( def _retry(
self, self, request: Request, reason: str | Exception | type[Exception]
request: Request,
reason: str | Exception | type[Exception],
spider: Spider,
) -> Request | None: ) -> Request | None:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
assert self.crawler.spider
return get_retry_request( return get_retry_request(
request, request,
reason=reason, reason=reason,
spider=spider, spider=self.crawler.spider,
max_retry_times=max_retry_times, max_retry_times=max_retry_times,
priority_adjust=priority_adjust, priority_adjust=priority_adjust,
) )

37
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -14,6 +14,7 @@ from twisted.internet.defer import Deferred
from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK from scrapy.http.request import NO_CALLBACK
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
@ -51,17 +52,18 @@ class RobotsTxtMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler) return cls(crawler)
async def process_request(self, request: Request, spider: Spider) -> None: @_warn_spider_arg
async def process_request(
self, request: Request, spider: Spider | None = None
) -> None:
if request.meta.get("dont_obey_robotstxt"): if request.meta.get("dont_obey_robotstxt"):
return return
if request.url.startswith("data:") or request.url.startswith("file:"): if request.url.startswith("data:") or request.url.startswith("file:"):
return return
rp = await self.robot_parser(request, spider) rp = await self.robot_parser(request)
self.process_request_2(rp, request, spider) self.process_request_2(rp, request)
def process_request_2( def process_request_2(self, rp: RobotParser | None, request: Request) -> None:
self, rp: RobotParser | None, request: Request, spider: Spider
) -> None:
if rp is None: if rp is None:
return return
@ -73,15 +75,13 @@ class RobotsTxtMiddleware:
logger.debug( logger.debug(
"Forbidden by robots.txt: %(request)s", "Forbidden by robots.txt: %(request)s",
{"request": request}, {"request": request},
extra={"spider": spider}, extra={"spider": self.crawler.spider},
) )
assert self.crawler.stats assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden") self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt") raise IgnoreRequest("Forbidden by robots.txt")
async def robot_parser( async def robot_parser(self, request: Request) -> RobotParser | None:
self, request: Request, spider: Spider
) -> RobotParser | None:
url = urlparse_cached(request) url = urlparse_cached(request)
netloc = url.netloc netloc = url.netloc
@ -100,7 +100,13 @@ class RobotsTxtMiddleware:
resp = await self.crawler.engine.download_async(robotsreq) resp = await self.crawler.engine.download_async(robotsreq)
self._parse_robots(resp, netloc) self._parse_robots(resp, netloc)
except Exception as e: except Exception as e:
self._logerror(e, robotsreq, spider) if not isinstance(e, IgnoreRequest):
logger.error(
"Error downloading %(request)s: %(f_exception)s",
{"request": request, "f_exception": e},
exc_info=True,
extra={"spider": self.crawler.spider},
)
self._robots_error(e, netloc) self._robots_error(e, netloc)
self.crawler.stats.inc_value("robotstxt/request_count") self.crawler.stats.inc_value("robotstxt/request_count")
@ -109,15 +115,6 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser) return await maybe_deferred_to_future(parser)
return parser return parser
def _logerror(self, exc: Exception, request: Request, spider: Spider) -> None:
if not isinstance(exc, IgnoreRequest):
logger.error(
"Error downloading %(request)s: %(f_exception)s",
{"request": request, "f_exception": exc},
exc_info=True, # noqa: LOG014
extra={"spider": spider},
)
def _parse_robots(self, response: Response, netloc: str) -> None: def _parse_robots(self, response: Response, netloc: str) -> None:
assert self.crawler.stats assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value("robotstxt/response_count")

10
scrapy/downloadermiddlewares/stats.py
View File

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from twisted.web import http from twisted.web import http
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr from scrapy.utils.request import request_httprepr
@ -45,8 +46,9 @@ class DownloaderStats:
assert crawler.stats assert crawler.stats
return cls(crawler.stats) return cls(crawler.stats)
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
self.stats.inc_value("downloader/request_count") self.stats.inc_value("downloader/request_count")
self.stats.inc_value(f"downloader/request_method_count/{request.method}") self.stats.inc_value(f"downloader/request_method_count/{request.method}")
@ -54,8 +56,9 @@ class DownloaderStats:
self.stats.inc_value("downloader/request_bytes", reqlen) self.stats.inc_value("downloader/request_bytes", reqlen)
return None return None
@_warn_spider_arg
def process_response( def process_response(
self, request: Request, response: Response, spider: Spider self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response: ) -> Request | Response:
self.stats.inc_value("downloader/response_count") self.stats.inc_value("downloader/response_count")
self.stats.inc_value(f"downloader/response_status_count/{response.status}") self.stats.inc_value(f"downloader/response_status_count/{response.status}")
@ -69,8 +72,9 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_bytes", reslen) self.stats.inc_value("downloader/response_bytes", reslen)
return response return response
@_warn_spider_arg
def process_exception( def process_exception(
self, request: Request, exception: Exception, spider: Spider self, request: Request, exception: Exception, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
ex_class = global_object_name(exception.__class__) ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count") self.stats.inc_value("downloader/exception_count")

4
scrapy/downloadermiddlewares/useragent.py
View File

@ -5,6 +5,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING: if TYPE_CHECKING:
# typing.Self requires Python 3.11 # typing.Self requires Python 3.11
@ -29,8 +30,9 @@ class UserAgentMiddleware:
def spider_opened(self, spider: Spider) -> None: def spider_opened(self, spider: Spider) -> None:
self.user_agent = getattr(spider, "user_agent", self.user_agent) self.user_agent = getattr(spider, "user_agent", self.user_agent)
@_warn_spider_arg
def process_request( def process_request(
self, request: Request, spider: Spider self, request: Request, spider: Spider | None = None
) -> Request | Response | None: ) -> Request | Response | None:
if self.user_agent: if self.user_agent:
request.headers.setdefault(b"User-Agent", self.user_agent) request.headers.setdefault(b"User-Agent", self.user_agent)

58
tests/test_downloadermiddleware.py
View File

@ -128,7 +128,7 @@ class TestResponseFromProcessRequest(TestManagerBase):
download_func = mock.MagicMock() download_func = mock.MagicMock()
class ResponseMiddleware: class ResponseMiddleware:
def process_request(self, request, spider): def process_request(self, request):
return resp return resp
async with self.get_mwman() as mwman: async with self.get_mwman() as mwman:
@ -151,11 +151,11 @@ class TestResponseFromProcessException(TestManagerBase):
raise ValueError("test") raise ValueError("test")
class ResponseMiddleware: class ResponseMiddleware:
def process_response(self, request, response, spider): def process_response(self, request, response):
calls.append("process_response") calls.append("process_response")
return resp return resp
def process_exception(self, request, exception, spider): def process_exception(self, request, exception):
calls.append("process_exception") calls.append("process_exception")
return resp return resp
@ -176,7 +176,7 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
class InvalidProcessRequestMiddleware: class InvalidProcessRequestMiddleware:
def process_request(self, request, spider): def process_request(self, request):
return 1 return 1
async with self.get_mwman() as mwman: async with self.get_mwman() as mwman:
@ -190,7 +190,7 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
class InvalidProcessResponseMiddleware: class InvalidProcessResponseMiddleware:
def process_response(self, request, response, spider): def process_response(self, request, response):
return 1 return 1
async with self.get_mwman() as mwman: async with self.get_mwman() as mwman:
@ -204,10 +204,10 @@ class TestInvalidOutput(TestManagerBase):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
class InvalidProcessExceptionMiddleware: class InvalidProcessExceptionMiddleware:
def process_request(self, request, spider): def process_request(self, request):
raise RuntimeError raise RuntimeError
def process_exception(self, request, exception, spider): def process_exception(self, request, exception):
return 1 return 1
async with self.get_mwman() as mwman: async with self.get_mwman() as mwman:
@ -229,7 +229,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
def cb(self, result): def cb(self, result):
return result return result
def process_request(self, request, spider): def process_request(self, request):
d = Deferred() d = Deferred()
d.addCallback(self.cb) d.addCallback(self.cb)
d.callback(resp) d.callback(resp)
@ -252,7 +252,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
download_func = mock.MagicMock() download_func = mock.MagicMock()
class CoroMiddleware: class CoroMiddleware:
async def process_request(self, request, spider): async def process_request(self, request):
await succeed(42) await succeed(42)
return resp return resp
@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
download_func = mock.MagicMock() download_func = mock.MagicMock()
class CoroMiddleware: class CoroMiddleware:
async def process_request(self, request, spider): async def process_request(self, request):
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
return await get_from_asyncio_queue(resp) return await get_from_asyncio_queue(resp)
@ -315,3 +315,41 @@ class TestDownloadDeprecated(TestManagerBase):
mwman.download(download_func, req, mwman.crawler.spider) mwman.download(download_func, req, mwman.crawler.spider)
) )
assert isinstance(ret, Response) assert isinstance(ret, Response)
class TestDeprecatedSpiderArg(TestManagerBase):
@deferred_f_from_coro_f
async def test_deprecated_spider_arg(self):
req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html")
download_func = mock.MagicMock()
class DeprecatedSpiderArgMiddleware:
def process_request(self, request, spider):
1 / 0
def process_response(self, request, response, spider):
return response
def process_exception(self, request, exception, spider):
return resp
async with self.get_mwman() as mwman:
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_request\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_response\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_exception\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgMiddleware())
result = await maybe_deferred_to_future(mwman.download(download_func, req))
assert result is resp
assert not download_func.called

158
tests/test_downloadermiddleware_cookies.py
View File

@ -8,9 +8,8 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
UNSET = object() UNSET = object()
@ -60,9 +59,10 @@ class TestCookiesMiddleware:
assert split_cookies(first) == split_cookies(second), msg assert split_cookies(first) == split_cookies(second), msg
def setup_method(self): def setup_method(self):
self.spider = Spider("foo") crawler = get_crawler(DefaultSpider)
self.mw = CookiesMiddleware() crawler.spider = crawler._create_spider()
self.redirect_middleware = RedirectMiddleware(settings=Settings()) self.mw = CookiesMiddleware.from_crawler(crawler)
self.redirect_middleware = RedirectMiddleware.from_crawler(crawler)
def teardown_method(self): def teardown_method(self):
del self.mw del self.mw
@ -70,15 +70,15 @@ class TestCookiesMiddleware:
def test_basic(self): def test_basic(self):
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers assert "Cookie" not in req.headers
headers = {"Set-Cookie": "C1=value1; path=/"} headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers) res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/") req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
assert req2.headers.get("Cookie") == b"C1=value1" assert req2.headers.get("Cookie") == b"C1=value1"
def test_setting_false_cookies_enabled(self): def test_setting_false_cookies_enabled(self):
@ -112,9 +112,9 @@ class TestCookiesMiddleware:
res = Response( res = Response(
"http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
) )
mw.process_response(req, res, crawler.spider) mw.process_response(req, res)
req2 = Request("http://scrapytest.org/sub1/") req2 = Request("http://scrapytest.org/sub1/")
mw.process_request(req2, crawler.spider) mw.process_request(req2)
log.check( log.check(
( (
@ -143,23 +143,23 @@ class TestCookiesMiddleware:
res = Response( res = Response(
"http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
) )
mw.process_response(req, res, crawler.spider) mw.process_response(req, res)
req2 = Request("http://scrapytest.org/sub1/") req2 = Request("http://scrapytest.org/sub1/")
mw.process_request(req2, crawler.spider) mw.process_request(req2)
log.check() log.check()
def test_do_not_break_on_non_utf8_header(self): def test_do_not_break_on_non_utf8_header(self):
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers assert "Cookie" not in req.headers
headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"} headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"}
res = Response("http://scrapytest.org/", headers=headers) res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/") req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
assert "Cookie" in req2.headers assert "Cookie" in req2.headers
def test_dont_merge_cookies(self): def test_dont_merge_cookies(self):
@ -167,11 +167,11 @@ class TestCookiesMiddleware:
headers = {"Set-Cookie": "C1=value1; path=/"} headers = {"Set-Cookie": "C1=value1; path=/"}
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
res = Response("http://scrapytest.org/", headers=headers) res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
# test Cookie header is not seted to request # test Cookie header is not seted to request
req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1}) req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1})
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert "Cookie" not in req.headers assert "Cookie" not in req.headers
# check that returned cookies are not merged back to jar # check that returned cookies are not merged back to jar
@ -179,16 +179,16 @@ class TestCookiesMiddleware:
"http://scrapytest.org/dontmerge", "http://scrapytest.org/dontmerge",
headers={"Set-Cookie": "dont=mergeme; path=/"}, headers={"Set-Cookie": "dont=mergeme; path=/"},
) )
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
# check that cookies are merged back # check that cookies are merged back
req = Request("http://scrapytest.org/mergeme") req = Request("http://scrapytest.org/mergeme")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"C1=value1" assert req.headers.get("Cookie") == b"C1=value1"
# check that cookies are merged when dont_merge_cookies is passed as 0 # check that cookies are merged when dont_merge_cookies is passed as 0
req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0})
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"C1=value1" assert req.headers.get("Cookie") == b"C1=value1"
def test_complex_cookies(self): def test_complex_cookies(self):
@ -216,11 +216,11 @@ class TestCookiesMiddleware:
] ]
req = Request("http://scrapytest.org/", cookies=cookies) req = Request("http://scrapytest.org/", cookies=cookies)
self.mw.process_request(req, self.spider) self.mw.process_request(req)
# embed C1 and C3 for scrapytest.org/foo # embed C1 and C3 for scrapytest.org/foo
req = Request("http://scrapytest.org/foo") req = Request("http://scrapytest.org/foo")
self.mw.process_request(req, self.spider) self.mw.process_request(req)
assert req.headers.get("Cookie") in ( assert req.headers.get("Cookie") in (
b"C1=value1; C3=value3", b"C1=value1; C3=value3",
b"C3=value3; C1=value1", b"C3=value3; C1=value1",
@ -228,25 +228,25 @@ class TestCookiesMiddleware:
# embed C2 for scrapytest.org/bar # embed C2 for scrapytest.org/bar
req = Request("http://scrapytest.org/bar") req = Request("http://scrapytest.org/bar")
self.mw.process_request(req, self.spider) self.mw.process_request(req)
assert req.headers.get("Cookie") == b"C2=value2" assert req.headers.get("Cookie") == b"C2=value2"
# embed nothing for scrapytest.org/baz # embed nothing for scrapytest.org/baz
req = Request("http://scrapytest.org/baz") req = Request("http://scrapytest.org/baz")
self.mw.process_request(req, self.spider) self.mw.process_request(req)
assert "Cookie" not in req.headers assert "Cookie" not in req.headers
def test_merge_request_cookies(self): def test_merge_request_cookies(self):
req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) req = Request("http://scrapytest.org/", cookies={"galleta": "salada"})
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"galleta=salada" assert req.headers.get("Cookie") == b"galleta=salada"
headers = {"Set-Cookie": "C1=value1; path=/"} headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers) res = Response("http://scrapytest.org/", headers=headers)
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/sub1/") req2 = Request("http://scrapytest.org/sub1/")
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual( self.assertCookieValEqual(
req2.headers.get("Cookie"), b"C1=value1; galleta=salada" req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
@ -258,15 +258,15 @@ class TestCookiesMiddleware:
cookies={"galleta": "salada"}, cookies={"galleta": "salada"},
meta={"cookiejar": "store1"}, meta={"cookiejar": "store1"},
) )
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers.get("Cookie") == b"galleta=salada" assert req.headers.get("Cookie") == b"galleta=salada"
headers = {"Set-Cookie": "C1=value1; path=/"} headers = {"Set-Cookie": "C1=value1; path=/"}
res = Response("http://scrapytest.org/", headers=headers, request=req) res = Response("http://scrapytest.org/", headers=headers, request=req)
assert self.mw.process_response(req, res, self.spider) is res assert self.mw.process_response(req, res) is res
req2 = Request("http://scrapytest.org/", meta=res.meta) req2 = Request("http://scrapytest.org/", meta=res.meta)
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual( self.assertCookieValEqual(
req2.headers.get("Cookie"), b"C1=value1; galleta=salada" req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
) )
@ -276,45 +276,45 @@ class TestCookiesMiddleware:
cookies={"galleta": "dulce"}, cookies={"galleta": "dulce"},
meta={"cookiejar": "store2"}, meta={"cookiejar": "store2"},
) )
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
assert req3.headers.get("Cookie") == b"galleta=dulce" assert req3.headers.get("Cookie") == b"galleta=dulce"
headers = {"Set-Cookie": "C2=value2; path=/"} headers = {"Set-Cookie": "C2=value2; path=/"}
res2 = Response("http://scrapytest.org/", headers=headers, request=req3) res2 = Response("http://scrapytest.org/", headers=headers, request=req3)
assert self.mw.process_response(req3, res2, self.spider) is res2 assert self.mw.process_response(req3, res2) is res2
req4 = Request("http://scrapytest.org/", meta=res2.meta) req4 = Request("http://scrapytest.org/", meta=res2.meta)
assert self.mw.process_request(req4, self.spider) is None assert self.mw.process_request(req4) is None
self.assertCookieValEqual( self.assertCookieValEqual(
req4.headers.get("Cookie"), b"C2=value2; galleta=dulce" req4.headers.get("Cookie"), b"C2=value2; galleta=dulce"
) )
# cookies from hosts with port # cookies from hosts with port
req5_1 = Request("http://scrapytest.org:1104/") req5_1 = Request("http://scrapytest.org:1104/")
assert self.mw.process_request(req5_1, self.spider) is None assert self.mw.process_request(req5_1) is None
headers = {"Set-Cookie": "C1=value1; path=/"} headers = {"Set-Cookie": "C1=value1; path=/"}
res5_1 = Response( res5_1 = Response(
"http://scrapytest.org:1104/", headers=headers, request=req5_1 "http://scrapytest.org:1104/", headers=headers, request=req5_1
) )
assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 assert self.mw.process_response(req5_1, res5_1) is res5_1
req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") req5_2 = Request("http://scrapytest.org:1104/some-redirected-path")
assert self.mw.process_request(req5_2, self.spider) is None assert self.mw.process_request(req5_2) is None
assert req5_2.headers.get("Cookie") == b"C1=value1" assert req5_2.headers.get("Cookie") == b"C1=value1"
req5_3 = Request("http://scrapytest.org/some-redirected-path") req5_3 = Request("http://scrapytest.org/some-redirected-path")
assert self.mw.process_request(req5_3, self.spider) is None assert self.mw.process_request(req5_3) is None
assert req5_3.headers.get("Cookie") == b"C1=value1" assert req5_3.headers.get("Cookie") == b"C1=value1"
# skip cookie retrieval for not http request # skip cookie retrieval for not http request
req6 = Request("file:///scrapy/sometempfile") req6 = Request("file:///scrapy/sometempfile")
assert self.mw.process_request(req6, self.spider) is None assert self.mw.process_request(req6) is None
assert req6.headers.get("Cookie") is None assert req6.headers.get("Cookie") is None
def test_local_domain(self): def test_local_domain(self):
request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) request = Request("http://example-host/", cookies={"currencyCookie": "USD"})
assert self.mw.process_request(request, self.spider) is None assert self.mw.process_request(request) is None
assert "Cookie" in request.headers assert "Cookie" in request.headers
assert request.headers["Cookie"] == b"currencyCookie=USD" assert request.headers["Cookie"] == b"currencyCookie=USD"
@ -324,15 +324,15 @@ class TestCookiesMiddleware:
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
# overwrite with values from 'cookies' request argument # overwrite with values from 'cookies' request argument
req1 = Request("http://example.org", cookies={"default": "something"}) req1 = Request("http://example.org", cookies={"default": "something"})
assert mw_default_headers.process_request(req1, self.spider) is None assert mw_default_headers.process_request(req1) is None
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
self.assertCookieValEqual( self.assertCookieValEqual(
req1.headers["Cookie"], b"default=something; asdf=qwerty" req1.headers["Cookie"], b"default=something; asdf=qwerty"
) )
# keep both # keep both
req2 = Request("http://example.com", cookies={"a": "b"}) req2 = Request("http://example.com", cookies={"a": "b"})
assert mw_default_headers.process_request(req2, self.spider) is None assert mw_default_headers.process_request(req2) is None
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual( self.assertCookieValEqual(
req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty" req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty"
) )
@ -341,13 +341,13 @@ class TestCookiesMiddleware:
def test_keep_cookie_header(self): def test_keep_cookie_header(self):
# keep only cookies from 'Cookie' request header # keep only cookies from 'Cookie' request header
req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"}) req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"})
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d") self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d")
# keep cookies from both 'Cookie' request header and 'cookies' keyword # keep cookies from both 'Cookie' request header and 'cookies' keyword
req2 = Request( req2 = Request(
"http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"} "http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"}
) )
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f") self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f")
# overwrite values from 'Cookie' request header with 'cookies' keyword # overwrite values from 'Cookie' request header with 'cookies' keyword
req3 = Request( req3 = Request(
@ -355,40 +355,40 @@ class TestCookiesMiddleware:
headers={"Cookie": "a=b; c=d"}, headers={"Cookie": "a=b; c=d"},
cookies={"a": "new", "e": "f"}, cookies={"a": "new", "e": "f"},
) )
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f") self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f")
def test_request_cookies_encoding(self): def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes # 1) UTF8-encoded bytes
req1 = Request("http://example.org", cookies={"a": "á".encode()}) req1 = Request("http://example.org", cookies={"a": "á".encode()})
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
# 2) Non UTF8-encoded bytes # 2) Non UTF8-encoded bytes
req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")}) req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")})
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
# 3) String # 3) String
req3 = Request("http://example.org", cookies={"a": "á"}) req3 = Request("http://example.org", cookies={"a": "á"})
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
@pytest.mark.xfail(reason="Cookie header is not currently being processed") @pytest.mark.xfail(reason="Cookie header is not currently being processed")
def test_request_headers_cookie_encoding(self): def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes # 1) UTF8-encoded bytes
req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
# 2) Non UTF8-encoded bytes # 2) Non UTF8-encoded bytes
req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")}) req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")})
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
# 3) String # 3) String
req3 = Request("http://example.org", headers={"Cookie": "a=á"}) req3 = Request("http://example.org", headers={"Cookie": "a=á"})
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
def test_invalid_cookies(self): def test_invalid_cookies(self):
@ -402,13 +402,13 @@ class TestCookiesMiddleware:
) as lc: ) as lc:
cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]
req1 = Request("http://example.org/1", cookies=cookies1) req1 = Request("http://example.org/1", cookies=cookies1)
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]
req2 = Request("http://example.org/2", cookies=cookies2) req2 = Request("http://example.org/2", cookies=cookies2)
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]
req3 = Request("http://example.org/3", cookies=cookies3) req3 = Request("http://example.org/3", cookies=cookies3)
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
lc.check( lc.check(
( (
"scrapy.downloadermiddlewares.cookies", "scrapy.downloadermiddlewares.cookies",
@ -436,22 +436,22 @@ class TestCookiesMiddleware:
def test_primitive_type_cookies(self): def test_primitive_type_cookies(self):
# Boolean # Boolean
req1 = Request("http://example.org", cookies={"a": True}) req1 = Request("http://example.org", cookies={"a": True})
assert self.mw.process_request(req1, self.spider) is None assert self.mw.process_request(req1) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=True") self.assertCookieValEqual(req1.headers["Cookie"], b"a=True")
# Float # Float
req2 = Request("http://example.org", cookies={"a": 9.5}) req2 = Request("http://example.org", cookies={"a": 9.5})
assert self.mw.process_request(req2, self.spider) is None assert self.mw.process_request(req2) is None
self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5") self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5")
# Integer # Integer
req3 = Request("http://example.org", cookies={"a": 10}) req3 = Request("http://example.org", cookies={"a": 10})
assert self.mw.process_request(req3, self.spider) is None assert self.mw.process_request(req3) is None
self.assertCookieValEqual(req3.headers["Cookie"], b"a=10") self.assertCookieValEqual(req3.headers["Cookie"], b"a=10")
# String # String
req4 = Request("http://example.org", cookies={"a": "b"}) req4 = Request("http://example.org", cookies={"a": "b"})
assert self.mw.process_request(req4, self.spider) is None assert self.mw.process_request(req4) is None
self.assertCookieValEqual(req4.headers["Cookie"], b"a=b") self.assertCookieValEqual(req4.headers["Cookie"], b"a=b")
def _test_cookie_redirect( def _test_cookie_redirect(
@ -471,7 +471,7 @@ class TestCookiesMiddleware:
target.setdefault("status", 301) target.setdefault("status", 301)
request1 = Request(cookies=input_cookies, **source) request1 = Request(cookies=input_cookies, **source)
self.mw.process_request(request1, self.spider) self.mw.process_request(request1)
cookies = request1.headers.get("Cookie") cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None) assert cookies == (b"a=b" if cookies1 else None)
@ -481,16 +481,12 @@ class TestCookiesMiddleware:
}, },
**target, **target,
) )
assert self.mw.process_response(request1, response, self.spider) == response assert self.mw.process_response(request1, response) == response
request2 = self.redirect_middleware.process_response( request2 = self.redirect_middleware.process_response(request1, response)
request1,
response,
self.spider,
)
assert isinstance(request2, Request) assert isinstance(request2, Request)
self.mw.process_request(request2, self.spider) self.mw.process_request(request2)
cookies = request2.headers.get("Cookie") cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None) assert cookies == (b"a=b" if cookies2 else None)
@ -564,11 +560,7 @@ class TestCookiesMiddleware:
**target, **target,
) )
request2 = self.redirect_middleware.process_response( request2 = self.redirect_middleware.process_response(request1, response)
request1,
response,
self.spider,
)
assert isinstance(request2, Request) assert isinstance(request2, Request)
cookies = request2.headers.get("Cookie") cookies = request2.headers.get("Cookie")
@ -620,12 +612,12 @@ class TestCookiesMiddleware:
] ]
request1 = Request(url1, cookies=input_cookies) request1 = Request(url1, cookies=input_cookies)
self.mw.process_request(request1, self.spider) self.mw.process_request(request1)
cookies = request1.headers.get("Cookie") cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None) assert cookies == (b"a=b" if cookies1 else None)
request2 = Request(url2) request2 = Request(url2)
self.mw.process_request(request2, self.spider) self.mw.process_request(request2)
cookies = request2.headers.get("Cookie") cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None) assert cookies == (b"a=b" if cookies2 else None)
@ -674,7 +666,7 @@ class TestCookiesMiddleware:
cookies, cookies,
): ):
request1 = Request(url1) request1 = Request(url1)
self.mw.process_request(request1, self.spider) self.mw.process_request(request1)
input_cookies = [ input_cookies = [
{ {
@ -688,10 +680,10 @@ class TestCookiesMiddleware:
"Set-Cookie": _cookies_to_set_cookie_list(input_cookies), "Set-Cookie": _cookies_to_set_cookie_list(input_cookies),
} }
response = Response(url1, status=200, headers=headers) response = Response(url1, status=200, headers=headers)
assert self.mw.process_response(request1, response, self.spider) == response assert self.mw.process_response(request1, response) == response
request2 = Request(url2) request2 = Request(url2)
self.mw.process_request(request2, self.spider) self.mw.process_request(request2)
actual_cookies = request2.headers.get("Cookie") actual_cookies = request2.headers.get("Cookie")
assert actual_cookies == (b"a=b" if cookies else None) assert actual_cookies == (b"a=b" if cookies else None)
@ -743,7 +735,7 @@ class TestCookiesMiddleware:
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
self.mw.process_request(request1, self.spider) self.mw.process_request(request1)
cookies = request1.headers.get("Cookie") cookies = request1.headers.get("Cookie")
assert cookies == (b"a=b" if cookies1 else None) assert cookies == (b"a=b" if cookies1 else None)
@ -752,18 +744,14 @@ class TestCookiesMiddleware:
headers={"Location": f"{to_scheme}://a.example"}, headers={"Location": f"{to_scheme}://a.example"},
status=301, status=301,
) )
assert self.mw.process_response(request1, response, self.spider) == response assert self.mw.process_response(request1, response) == response
request2 = self.redirect_middleware.process_response( request2 = self.redirect_middleware.process_response(request1, response)
request1,
response,
self.spider,
)
assert isinstance(request2, Request) assert isinstance(request2, Request)
cookies = request2.headers.get("Cookie") cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies2 else None) assert cookies == (b"a=b" if cookies2 else None)
self.mw.process_request(request2, self.spider) self.mw.process_request(request2)
cookies = request2.headers.get("Cookie") cookies = request2.headers.get("Cookie")
assert cookies == (b"a=b" if cookies3 else None) assert cookies == (b"a=b" if cookies3 else None)

13
tests/test_downloadermiddleware_defaultheaders.py
View File

@ -6,28 +6,27 @@ from scrapy.utils.test import get_crawler
class TestDefaultHeadersMiddleware: class TestDefaultHeadersMiddleware:
def get_defaults_spider_mw(self): def get_defaults_mw(self):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider("foo")
defaults = { defaults = {
to_bytes(k): [to_bytes(v)] to_bytes(k): [to_bytes(v)]
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
} }
return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
def test_process_request(self): def test_process_request(self):
defaults, spider, mw = self.get_defaults_spider_mw() defaults, mw = self.get_defaults_mw()
req = Request("http://www.scrapytest.org") req = Request("http://www.scrapytest.org")
mw.process_request(req, spider) mw.process_request(req)
assert req.headers == defaults assert req.headers == defaults
def test_update_headers(self): def test_update_headers(self):
defaults, spider, mw = self.get_defaults_spider_mw() defaults, mw = self.get_defaults_mw()
headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
req = Request("http://www.scrapytest.org", headers=headers) req = Request("http://www.scrapytest.org", headers=headers)
assert req.headers == bytes_headers assert req.headers == bytes_headers
mw.process_request(req, spider) mw.process_request(req)
defaults.update(bytes_headers) defaults.update(bytes_headers)
assert req.headers == defaults assert req.headers == defaults

8
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -14,20 +14,20 @@ class TestDownloadTimeoutMiddleware:
def test_default_download_timeout(self): def test_default_download_timeout(self):
req, spider, mw = self.get_request_spider_mw() req, spider, mw = self.get_request_spider_mw()
mw.spider_opened(spider) mw.spider_opened(spider)
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 180 assert req.meta.get("download_timeout") == 180
def test_string_download_timeout(self): def test_string_download_timeout(self):
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
mw.spider_opened(spider) mw.spider_opened(spider)
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 20.1 assert req.meta.get("download_timeout") == 20.1
def test_spider_has_download_timeout(self): def test_spider_has_download_timeout(self):
req, spider, mw = self.get_request_spider_mw() req, spider, mw = self.get_request_spider_mw()
spider.download_timeout = 2 spider.download_timeout = 2
mw.spider_opened(spider) mw.spider_opened(spider)
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 2 assert req.meta.get("download_timeout") == 2
def test_request_has_download_timeout(self): def test_request_has_download_timeout(self):
@ -35,5 +35,5 @@ class TestDownloadTimeoutMiddleware:
spider.download_timeout = 2 spider.download_timeout = 2
mw.spider_opened(spider) mw.spider_opened(spider)
req.meta["download_timeout"] = 1 req.meta["download_timeout"] = 1
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 1 assert req.meta.get("download_timeout") == 1

20
tests/test_downloadermiddleware_httpauth.py
View File

@ -36,48 +36,48 @@ class TestHttpAuthMiddlewareLegacy:
class TestHttpAuthMiddleware: class TestHttpAuthMiddleware:
def setup_method(self): def setup_method(self):
self.mw = HttpAuthMiddleware() self.mw = HttpAuthMiddleware()
self.spider = DomainSpider("foo") spider = DomainSpider("foo")
self.mw.spider_opened(self.spider) self.mw.spider_opened(spider)
def teardown_method(self): def teardown_method(self):
del self.mw del self.mw
def test_no_auth(self): def test_no_auth(self):
req = Request("http://example-noauth.com/") req = Request("http://example-noauth.com/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert "Authorization" not in req.headers assert "Authorization" not in req.headers
def test_auth_domain(self): def test_auth_domain(self):
req = Request("http://example.com/") req = Request("http://example.com/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar") assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_subdomain(self): def test_auth_subdomain(self):
req = Request("http://foo.example.com/") req = Request("http://foo.example.com/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar") assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_already_set(self): def test_auth_already_set(self):
req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == b"Digest 123" assert req.headers["Authorization"] == b"Digest 123"
class TestHttpAuthAnyMiddleware: class TestHttpAuthAnyMiddleware:
def setup_method(self): def setup_method(self):
self.mw = HttpAuthMiddleware() self.mw = HttpAuthMiddleware()
self.spider = AnyDomainSpider("foo") spider = AnyDomainSpider("foo")
self.mw.spider_opened(self.spider) self.mw.spider_opened(spider)
def teardown_method(self): def teardown_method(self):
del self.mw del self.mw
def test_auth(self): def test_auth(self):
req = Request("http://example.com/") req = Request("http://example.com/")
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar") assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_already_set(self): def test_auth_already_set(self):
req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == b"Digest 123" assert req.headers["Authorization"] == b"Digest 123"

184
tests/test_downloadermiddleware_httpcache.py
View File

@ -1,28 +1,36 @@
from __future__ import annotations
import email.utils import email.utils
import shutil import shutil
import tempfile import tempfile
import time import time
from contextlib import contextmanager from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import pytest import pytest
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest from scrapy.exceptions import IgnoreRequest
from scrapy.http import HtmlResponse, Request, Response from scrapy.http import HtmlResponse, Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Generator
from scrapy.crawler import Crawler
class TestBase: class TestBase:
"""Base class with common setup and helper methods.""" """Base class with common setup and helper methods."""
policy_class: str
storage_class: str
def setup_method(self): def setup_method(self):
self.yesterday = email.utils.formatdate(time.time() - 86400) self.yesterday = email.utils.formatdate(time.time() - 86400)
self.today = email.utils.formatdate() self.today = email.utils.formatdate()
self.tomorrow = email.utils.formatdate(time.time() + 86400) self.tomorrow = email.utils.formatdate(time.time() + 86400)
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("example.com")
self.tmpdir = tempfile.mkdtemp() self.tmpdir = tempfile.mkdtemp()
self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.request = Request("http://www.example.com", headers={"User-Agent": "test"})
self.response = Response( self.response = Response(
@ -31,13 +39,11 @@ class TestBase:
body=b"test body", body=b"test body",
status=202, status=202,
) )
self.crawler.stats.open_spider()
def teardown_method(self): def teardown_method(self):
self.crawler.stats.close_spider()
shutil.rmtree(self.tmpdir) shutil.rmtree(self.tmpdir)
def _get_settings(self, **new_settings): def _get_settings(self, **new_settings: Any) -> dict[str, Any]:
settings = { settings = {
"HTTPCACHE_ENABLED": True, "HTTPCACHE_ENABLED": True,
"HTTPCACHE_DIR": self.tmpdir, "HTTPCACHE_DIR": self.tmpdir,
@ -47,27 +53,35 @@ class TestBase:
"HTTPCACHE_STORAGE": self.storage_class, "HTTPCACHE_STORAGE": self.storage_class,
} }
settings.update(new_settings) settings.update(new_settings)
return Settings(settings) return settings
@contextmanager @contextmanager
def _storage(self, **new_settings): def _get_crawler(self, **new_settings: Any) -> Generator[Crawler]:
with self._middleware(**new_settings) as mw:
yield mw.storage
@contextmanager
def _policy(self, **new_settings):
with self._middleware(**new_settings) as mw:
yield mw.policy
@contextmanager
def _middleware(self, **new_settings):
settings = self._get_settings(**new_settings) settings = self._get_settings(**new_settings)
mw = HttpCacheMiddleware(settings, self.crawler.stats) crawler = get_crawler(Spider, settings)
mw.spider_opened(self.spider) crawler.spider = crawler._create_spider("example.com")
assert crawler.stats
crawler.stats.open_spider()
try: try:
yield mw yield crawler
finally: finally:
mw.spider_closed(self.spider) crawler.stats.close_spider()
@contextmanager
def _storage(self, **new_settings: Any):
with self._middleware(**new_settings) as mw:
yield mw.storage, mw.crawler
@contextmanager
def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]:
with self._get_crawler(**new_settings) as crawler:
assert crawler.spider
mw = HttpCacheMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
try:
yield mw
finally:
mw.spider_closed(crawler.spider)
def assertEqualResponse(self, response1, response2): def assertEqualResponse(self, response1, response2):
assert response1.url == response2.url assert response1.url == response2.url
@ -94,37 +108,37 @@ class StorageTestMixin:
"""Mixin containing storage-specific test methods.""" """Mixin containing storage-specific test methods."""
def test_storage(self): def test_storage(self):
with self._storage() as storage: with self._storage() as (storage, crawler):
request2 = self.request.copy() request2 = self.request.copy()
assert storage.retrieve_response(self.spider, request2) is None assert storage.retrieve_response(crawler.spider, request2) is None
storage.store_response(self.spider, self.request, self.response) storage.store_response(crawler.spider, self.request, self.response)
response2 = storage.retrieve_response(self.spider, request2) response2 = storage.retrieve_response(crawler.spider, request2)
assert isinstance(response2, HtmlResponse) # content-type header assert isinstance(response2, HtmlResponse) # content-type header
self.assertEqualResponse(self.response, response2) self.assertEqualResponse(self.response, response2)
time.sleep(2) # wait for cache to expire time.sleep(2) # wait for cache to expire
assert storage.retrieve_response(self.spider, request2) is None assert storage.retrieve_response(crawler.spider, request2) is None
def test_storage_never_expire(self): def test_storage_never_expire(self):
with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler):
assert storage.retrieve_response(self.spider, self.request) is None assert storage.retrieve_response(crawler.spider, self.request) is None
storage.store_response(self.spider, self.request, self.response) storage.store_response(crawler.spider, self.request, self.response)
time.sleep(0.5) # give the chance to expire time.sleep(0.5) # give the chance to expire
assert storage.retrieve_response(self.spider, self.request) assert storage.retrieve_response(crawler.spider, self.request)
def test_storage_no_content_type_header(self): def test_storage_no_content_type_header(self):
"""Test that the response body is used to get the right response class """Test that the response body is used to get the right response class
even if there is no Content-Type header""" even if there is no Content-Type header"""
with self._storage() as storage: with self._storage() as (storage, crawler):
assert storage.retrieve_response(self.spider, self.request) is None assert storage.retrieve_response(crawler.spider, self.request) is None
response = Response( response = Response(
"http://www.example.com", "http://www.example.com",
body=b"<!DOCTYPE html>\n<title>.</title>", body=b"<!DOCTYPE html>\n<title>.</title>",
status=202, status=202,
) )
storage.store_response(self.spider, self.request, response) storage.store_response(crawler.spider, self.request, response)
cached_response = storage.retrieve_response(self.spider, self.request) cached_response = storage.retrieve_response(crawler.spider, self.request)
assert isinstance(cached_response, HtmlResponse) assert isinstance(cached_response, HtmlResponse)
self.assertEqualResponse(response, cached_response) self.assertEqualResponse(response, cached_response)
@ -135,15 +149,15 @@ class PolicyTestMixin:
def test_dont_cache(self): def test_dont_cache(self):
with self._middleware() as mw: with self._middleware() as mw:
self.request.meta["dont_cache"] = True self.request.meta["dont_cache"] = True
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
assert mw.storage.retrieve_response(self.spider, self.request) is None assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None
with self._middleware() as mw: with self._middleware() as mw:
self.request.meta["dont_cache"] = False self.request.meta["dont_cache"] = False
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
if mw.policy.should_cache_response(self.response, self.request): if mw.policy.should_cache_response(self.response, self.request):
assert isinstance( assert isinstance(
mw.storage.retrieve_response(self.spider, self.request), mw.storage.retrieve_response(mw.crawler.spider, self.request),
self.response.__class__, self.response.__class__,
) )
@ -153,9 +167,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
def test_middleware(self): def test_middleware(self):
with self._middleware() as mw: with self._middleware() as mw:
assert mw.process_request(self.request, self.spider) is None assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
response = mw.process_request(self.request, self.spider) response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse) assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response) self.assertEqualResponse(self.response, response)
assert "cached" in response.flags assert "cached" in response.flags
@ -164,9 +178,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
with self._middleware() as mw: with self._middleware() as mw:
req = Request("http://host.com/path") req = Request("http://host.com/path")
res = Response("http://host2.net/test.html") res = Response("http://host2.net/test.html")
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
mw.process_response(req, res, self.spider) mw.process_response(req, res)
cached = mw.process_request(req, self.spider) cached = mw.process_request(req)
assert isinstance(cached, Response) assert isinstance(cached, Response)
self.assertEqualResponse(res, cached) self.assertEqualResponse(res, cached)
assert "cached" in cached.flags assert "cached" in cached.flags
@ -174,9 +188,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):
def test_middleware_ignore_missing(self): def test_middleware_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_request(self.request, self.spider) mw.process_request(self.request)
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
response = mw.process_request(self.request, self.spider) response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse) assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response) self.assertEqualResponse(self.response, response)
assert "cached" in response.flags assert "cached" in response.flags
@ -185,10 +199,10 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# http responses are cached by default # http responses are cached by default
req, res = Request("http://test.com/"), Response("http://test.com/") req, res = Request("http://test.com/"), Response("http://test.com/")
with self._middleware() as mw: with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
mw.process_response(req, res, self.spider) mw.process_response(req, res)
cached = mw.process_request(req, self.spider) cached = mw.process_request(req)
assert isinstance(cached, Response), type(cached) assert isinstance(cached, Response), type(cached)
self.assertEqualResponse(res, cached) self.assertEqualResponse(res, cached)
assert "cached" in cached.flags assert "cached" in cached.flags
@ -196,19 +210,19 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# file response is not cached by default # file response is not cached by default
req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt") req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt")
with self._middleware() as mw: with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
mw.process_response(req, res, self.spider) mw.process_response(req, res)
assert mw.storage.retrieve_response(self.spider, req) is None assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
# s3 scheme response is cached by default # s3 scheme response is cached by default
req, res = Request("s3://bucket/key"), Response("http://bucket/key") req, res = Request("s3://bucket/key"), Response("http://bucket/key")
with self._middleware() as mw: with self._middleware() as mw:
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
mw.process_response(req, res, self.spider) mw.process_response(req, res)
cached = mw.process_request(req, self.spider) cached = mw.process_request(req)
assert isinstance(cached, Response), type(cached) assert isinstance(cached, Response), type(cached)
self.assertEqualResponse(res, cached) self.assertEqualResponse(res, cached)
assert "cached" in cached.flags assert "cached" in cached.flags
@ -216,25 +230,25 @@ class DummyPolicyTestMixin(PolicyTestMixin):
# ignore s3 scheme # ignore s3 scheme
req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
mw.process_response(req, res, self.spider) mw.process_response(req, res)
assert mw.storage.retrieve_response(self.spider, req) is None assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
assert mw.process_request(req, self.spider) is None assert mw.process_request(req) is None
def test_middleware_ignore_http_codes(self): def test_middleware_ignore_http_codes(self):
# test response is not cached # test response is not cached
with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw: with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw:
assert mw.process_request(self.request, self.spider) is None assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
assert mw.storage.retrieve_response(self.spider, self.request) is None assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None
assert mw.process_request(self.request, self.spider) is None assert mw.process_request(self.request) is None
# test response is cached # test response is cached
with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw: with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw:
mw.process_response(self.request, self.response, self.spider) mw.process_response(self.request, self.response)
response = mw.process_request(self.request, self.spider) response = mw.process_request(self.request)
assert isinstance(response, HtmlResponse) assert isinstance(response, HtmlResponse)
self.assertEqualResponse(self.response, response) self.assertEqualResponse(self.response, response)
assert "cached" in response.flags assert "cached" in response.flags
@ -243,14 +257,18 @@ class DummyPolicyTestMixin(PolicyTestMixin):
class RFC2616PolicyTestMixin(PolicyTestMixin): class RFC2616PolicyTestMixin(PolicyTestMixin):
"""Mixin containing RFC2616 policy specific test methods.""" """Mixin containing RFC2616 policy specific test methods."""
def _process_requestresponse(self, mw, request, response): @staticmethod
def _process_requestresponse(
mw: HttpCacheMiddleware, request: Request, response: Response | None
) -> Response | Request:
result = None result = None
try: try:
result = mw.process_request(request, self.spider) result = mw.process_request(request)
if result: if result:
assert isinstance(result, (Request, Response)) assert isinstance(result, (Request, Response))
return result return result
result = mw.process_response(request, response, self.spider) assert response is not None
result = mw.process_response(request, response)
assert isinstance(result, Response) assert isinstance(result, Response)
return result return result
except Exception: except Exception:
@ -270,11 +288,11 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# response for a request with no-store must not be cached # response for a request with no-store must not be cached
res1 = self._process_requestresponse(mw, req1, res0) res1 = self._process_requestresponse(mw, req1, res0)
self.assertEqualResponse(res1, res0) self.assertEqualResponse(res1, res0)
assert mw.storage.retrieve_response(self.spider, req1) is None assert mw.storage.retrieve_response(mw.crawler.spider, req1) is None
# Re-do request without no-store and expect it to be cached # Re-do request without no-store and expect it to be cached
res2 = self._process_requestresponse(mw, req0, res0) res2 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res2.flags assert "cached" not in res2.flags
res3 = mw.process_request(req0, self.spider) res3 = mw.process_request(req0)
assert "cached" in res3.flags assert "cached" in res3.flags
self.assertEqualResponse(res2, res3) self.assertEqualResponse(res2, res3)
# request with no-cache directive must not return cached response # request with no-cache directive must not return cached response
@ -330,7 +348,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
) )
self.assertEqualResponse(res1, res0) self.assertEqualResponse(res1, res0)
self.assertEqualResponse(res2, res0) self.assertEqualResponse(res2, res0)
resc = mw.storage.retrieve_response(self.spider, req0) resc = mw.storage.retrieve_response(mw.crawler.spider, req0)
if shouldcache: if shouldcache:
self.assertEqualResponse(resc, res1) self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags assert "cached" in res2.flags
@ -354,7 +372,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
) )
self.assertEqualResponse(res1, res0) self.assertEqualResponse(res1, res0)
self.assertEqualResponse(res2, res0) self.assertEqualResponse(res2, res0)
resc = mw.storage.retrieve_response(self.spider, req0) resc = mw.storage.retrieve_response(mw.crawler.spider, req0)
if shouldcache: if shouldcache:
self.assertEqualResponse(resc, res1) self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags assert "cached" in res2.flags
@ -421,7 +439,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# validate cached response if request max-age set as 0 # validate cached response if request max-age set as 0
req1 = req0.replace(headers={"Cache-Control": "max-age=0"}) req1 = req0.replace(headers={"Cache-Control": "max-age=0"})
res304 = res0.replace(status=304) res304 = res0.replace(status=304)
assert mw.process_request(req1, self.spider) is None assert mw.process_request(req1) is None
res3 = self._process_requestresponse(mw, req1, res304) res3 = self._process_requestresponse(mw, req1, res304)
self.assertEqualResponse(res1, res3) self.assertEqualResponse(res1, res3)
assert "cached" in res3.flags assert "cached" in res3.flags
@ -513,14 +531,14 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
self._process_requestresponse(mw, req0, res0) self._process_requestresponse(mw, req0, res0)
for e in mw.DOWNLOAD_EXCEPTIONS: for e in mw.DOWNLOAD_EXCEPTIONS:
# Simulate encountering an error on download attempts # Simulate encountering an error on download attempts
assert mw.process_request(req0, self.spider) is None assert mw.process_request(req0) is None
res1 = mw.process_exception(req0, e("foo"), self.spider) res1 = mw.process_exception(req0, e("foo"))
# Use cached response as recovery # Use cached response as recovery
assert "cached" in res1.flags assert "cached" in res1.flags
self.assertEqualResponse(res0, res1) self.assertEqualResponse(res0, res1)
# Do not use cached response for unhandled exceptions # Do not use cached response for unhandled exceptions
mw.process_request(req0, self.spider) mw.process_request(req0)
assert mw.process_exception(req0, Exception("foo"), self.spider) is None assert mw.process_exception(req0, Exception("foo")) is None
def test_ignore_response_cache_controls(self): def test_ignore_response_cache_controls(self):
sampledata = [ sampledata = [
@ -578,17 +596,17 @@ class TestDbmStorageWithRFC2616Policy(
class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy): class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy):
dbm_module = "tests.mocks.dummydbm" dbm_module = "tests.mocks.dummydbm"
def _get_settings(self, **new_settings): def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)
return super()._get_settings(**new_settings) return super()._get_settings(**new_settings)
def test_custom_dbm_module_loaded(self): def test_custom_dbm_module_loaded(self):
# make sure our dbm module has been loaded # make sure our dbm module has been loaded
with self._storage() as storage: with self._storage() as (storage, _):
assert storage.dbmodule.__name__ == self.dbm_module assert storage.dbmodule.__name__ == self.dbm_module
class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy): class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy):
def _get_settings(self, **new_settings): def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_GZIP", True) new_settings.setdefault("HTTPCACHE_GZIP", True)
return super()._get_settings(**new_settings) return super()._get_settings(**new_settings)

61
tests/test_downloadermiddleware_httpcompression.py
View File

@ -70,7 +70,6 @@ def _skip_if_no_zstd() -> None:
class TestHttpCompression: class TestHttpCompression:
def setup_method(self): def setup_method(self):
self.crawler = get_crawler(Spider) self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider() self.crawler.stats.open_spider()
@ -124,7 +123,7 @@ class TestHttpCompression:
def test_process_request(self): def test_process_request(self):
request = Request("http://scrapytest.org") request = Request("http://scrapytest.org")
assert "Accept-Encoding" not in request.headers assert "Accept-Encoding" not in request.headers
self.mw.process_request(request, self.spider) self.mw.process_request(request)
assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS) assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)
def test_process_response_gzip(self): def test_process_response_gzip(self):
@ -132,7 +131,7 @@ class TestHttpCompression:
request = response.request request = response.request
assert response.headers["Content-Encoding"] == b"gzip" assert response.headers["Content-Encoding"] == b"gzip"
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -145,7 +144,7 @@ class TestHttpCompression:
response = self._getresponse("br") response = self._getresponse("br")
request = response.request request = response.request
assert response.headers["Content-Encoding"] == b"br" assert response.headers["Content-Encoding"] == b"br"
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -172,7 +171,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",
@ -197,7 +196,7 @@ class TestHttpCompression:
response = self._getresponse(check_key) response = self._getresponse(check_key)
request = response.request request = response.request
assert response.headers["Content-Encoding"] == b"zstd" assert response.headers["Content-Encoding"] == b"zstd"
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
if raw_content is None: if raw_content is None:
raw_content = newresponse.body raw_content = newresponse.body
else: else:
@ -221,7 +220,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",
@ -241,7 +240,7 @@ class TestHttpCompression:
request = response.request request = response.request
assert response.headers["Content-Encoding"] == b"deflate" assert response.headers["Content-Encoding"] == b"deflate"
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -253,7 +252,7 @@ class TestHttpCompression:
request = response.request request = response.request
assert response.headers["Content-Encoding"] == b"deflate" assert response.headers["Content-Encoding"] == b"deflate"
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -265,7 +264,7 @@ class TestHttpCompression:
request = Request("http://scrapytest.org") request = Request("http://scrapytest.org")
assert not response.headers.get("Content-Encoding") assert not response.headers.get("Content-Encoding")
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is response assert newresponse is response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_count", None)
@ -275,14 +274,14 @@ class TestHttpCompression:
response = self._getresponse("gzip") response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"] response.headers["Content-Encoding"] = ["uuencode", "gzip"]
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"] assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]
def test_multi_compression_single_header(self): def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate") response = self._getresponse("gzip-deflate")
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
@ -296,7 +295,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",
@ -314,7 +313,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate") response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"] response.headers["Content-Encoding"] = ["gzip", "deflate"]
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
@ -323,7 +322,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate") response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"] response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@ -331,7 +330,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate-gzip") response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"] response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
@ -340,7 +339,7 @@ class TestHttpCompression:
response = self._getresponse("gzip-deflate") response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"] response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@ -362,7 +361,7 @@ class TestHttpCompression:
) )
request = Request("http://www.example.com/") request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse) assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312") assert newresponse.encoding == resolve_encoding("gb2312")
@ -387,7 +386,7 @@ class TestHttpCompression:
) )
request = Request("http://www.example.com/") request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse) assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312") assert newresponse.encoding == resolve_encoding("gb2312")
@ -410,7 +409,7 @@ class TestHttpCompression:
) )
request = Request("http://www.example.com/index") request = Request("http://www.example.com/index")
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, respcls) assert isinstance(newresponse, respcls)
assert newresponse.body == plainbody assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312") assert newresponse.encoding == resolve_encoding("gb2312")
@ -422,7 +421,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "application/gzip" response.headers["Content-Type"] = "application/gzip"
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -434,7 +433,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "application/octet-stream" response.headers["Content-Type"] = "application/octet-stream"
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -446,7 +445,7 @@ class TestHttpCompression:
response.headers["Content-Type"] = "binary/octet-stream" response.headers["Content-Type"] = "binary/octet-stream"
request = response.request request = response.request
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is not response assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE") assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers assert "Content-Encoding" not in newresponse.headers
@ -494,7 +493,7 @@ class TestHttpCompression:
) )
request = Request("http://www.example.com/") request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert gunzip(newresponse.body) == plainbody assert gunzip(newresponse.body) == plainbody
self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 230) self.assertStatsEqual("httpcompression/response_bytes", 230)
@ -505,7 +504,7 @@ class TestHttpCompression:
request = response.request request = response.request
request.method = "HEAD" request.method = "HEAD"
response = response.replace(body=None) response = response.replace(body=None)
newresponse = self.mw.process_response(request, response, self.spider) newresponse = self.mw.process_response(request, response)
assert newresponse is response assert newresponse is response
assert response.body == b"" assert response.body == b""
self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_count", None)
@ -520,7 +519,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}") response = self._getresponse(f"bomb-{compression_id}")
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
def test_compression_bomb_setting_br(self): def test_compression_bomb_setting_br(self):
_skip_if_no_br() _skip_if_no_br()
@ -549,7 +548,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}") response = self._getresponse(f"bomb-{compression_id}")
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self): def test_compression_bomb_spider_attr_br(self):
@ -580,7 +579,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}") response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 10_000_000 response.meta["download_maxsize"] = 10_000_000
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
def test_compression_bomb_request_meta_br(self): def test_compression_bomb_request_meta_br(self):
_skip_if_no_br() _skip_if_no_br()
@ -611,7 +610,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",
@ -655,7 +654,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",
@ -701,7 +700,7 @@ class TestHttpCompression:
propagate=False, propagate=False,
level=WARNING, level=WARNING,
) as log: ) as log:
mw.process_response(response.request, response, spider) mw.process_response(response.request, response)
log.check( log.check(
( (
"scrapy.downloadermiddlewares.httpcompression", "scrapy.downloadermiddlewares.httpcompression",

100
tests/test_downloadermiddleware_httpproxy.py
View File

@ -8,8 +8,6 @@ from scrapy.http import Request
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
spider = Spider("foo")
class TestHttpProxyMiddleware: class TestHttpProxyMiddleware:
failureException = AssertionError # type: ignore[assignment] failureException = AssertionError # type: ignore[assignment]
@ -31,7 +29,7 @@ class TestHttpProxyMiddleware:
for url in ("http://e.com", "https://e.com", "file:///tmp/a"): for url in ("http://e.com", "https://e.com", "file:///tmp/a"):
req = Request(url) req = Request(url)
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.url == url assert req.url == url
assert req.meta == {} assert req.meta == {}
@ -47,7 +45,7 @@ class TestHttpProxyMiddleware:
("file://tmp/a", None), ("file://tmp/a", None),
]: ]:
req = Request(url) req = Request(url)
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.url == url assert req.url == url
assert req.meta.get("proxy") == proxy assert req.meta.get("proxy") == proxy
@ -55,14 +53,14 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://proxy.com" os.environ["http_proxy"] = "https://proxy.com"
mw = HttpProxyMiddleware() mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"}) req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"})
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta == {"proxy": "https://new.proxy:3128"} assert req.meta == {"proxy": "https://new.proxy:3128"}
def test_proxy_auth(self): def test_proxy_auth(self):
os.environ["http_proxy"] = "https://user:pass@proxy:3128" os.environ["http_proxy"] = "https://user:pass@proxy:3128"
mw = HttpProxyMiddleware() mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org") req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjpwYXNz" assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjpwYXNz"
# proxy from request.meta # proxy from request.meta
@ -70,7 +68,7 @@ class TestHttpProxyMiddleware:
"http://scrapytest.org", "http://scrapytest.org",
meta={"proxy": "https://username:password@proxy:3128"}, meta={"proxy": "https://username:password@proxy:3128"},
) )
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert ( assert (
req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=" req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ="
@ -80,14 +78,14 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://user:@proxy:3128" os.environ["http_proxy"] = "https://user:@proxy:3128"
mw = HttpProxyMiddleware() mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org") req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjo=" assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjo="
# proxy from request.meta # proxy from request.meta
req = Request( req = Request(
"http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"} "http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"}
) )
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6" assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6"
@ -96,7 +94,7 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128" os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128"
mw = HttpProxyMiddleware(auth_encoding="utf-8") mw = HttpProxyMiddleware(auth_encoding="utf-8")
req = Request("http://scrapytest.org") req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic bcOhbjpwYXNz" assert req.headers.get("Proxy-Authorization") == b"Basic bcOhbjpwYXNz"
@ -104,14 +102,14 @@ class TestHttpProxyMiddleware:
req = Request( req = Request(
"http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"}
) )
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic w7xzZXI6cGFzcw==" assert req.headers.get("Proxy-Authorization") == b"Basic w7xzZXI6cGFzcw=="
# default latin-1 encoding # default latin-1 encoding
mw = HttpProxyMiddleware(auth_encoding="latin-1") mw = HttpProxyMiddleware(auth_encoding="latin-1")
req = Request("http://scrapytest.org") req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic beFuOnBhc3M=" assert req.headers.get("Proxy-Authorization") == b"Basic beFuOnBhc3M="
@ -119,7 +117,7 @@ class TestHttpProxyMiddleware:
req = Request( req = Request(
"http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"}
) )
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta["proxy"] == "https://proxy:3128" assert req.meta["proxy"] == "https://proxy:3128"
assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz" assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz"
@ -127,7 +125,7 @@ class TestHttpProxyMiddleware:
os.environ["http_proxy"] = "https://proxy.for.http:3128" os.environ["http_proxy"] = "https://proxy.for.http:3128"
mw = HttpProxyMiddleware() mw = HttpProxyMiddleware()
req = Request("http://noproxy.com", meta={"proxy": None}) req = Request("http://noproxy.com", meta={"proxy": None})
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert "proxy" in req.meta assert "proxy" in req.meta
assert req.meta["proxy"] is None assert req.meta["proxy"] is None
@ -137,23 +135,23 @@ class TestHttpProxyMiddleware:
os.environ["no_proxy"] = "*" os.environ["no_proxy"] = "*"
req = Request("http://noproxy.com") req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert "proxy" not in req.meta assert "proxy" not in req.meta
os.environ["no_proxy"] = "other.com" os.environ["no_proxy"] = "other.com"
req = Request("http://noproxy.com") req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert "proxy" in req.meta assert "proxy" in req.meta
os.environ["no_proxy"] = "other.com,noproxy.com" os.environ["no_proxy"] = "other.com,noproxy.com"
req = Request("http://noproxy.com") req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert "proxy" not in req.meta assert "proxy" not in req.meta
# proxy from meta['proxy'] takes precedence # proxy from meta['proxy'] takes precedence
os.environ["no_proxy"] = "*" os.environ["no_proxy"] = "*"
req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"}) req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"})
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.meta == {"proxy": "http://proxy.com"} assert req.meta == {"proxy": "http://proxy.com"}
def test_no_proxy_invalid_values(self): def test_no_proxy_invalid_values(self):
@ -166,18 +164,18 @@ class TestHttpProxyMiddleware:
def test_add_proxy_without_credentials(self): def test_add_proxy_without_credentials(self):
middleware = HttpProxyMiddleware() middleware = HttpProxyMiddleware()
request = Request("https://example.com") request = Request("https://example.com")
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.com" request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
def test_add_proxy_with_credentials(self): def test_add_proxy_with_credentials(self):
middleware = HttpProxyMiddleware() middleware = HttpProxyMiddleware()
request = Request("https://example.com") request = Request("https://example.com")
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.com" request.meta["proxy"] = "https://user1:password1@example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user1", "user1",
@ -191,9 +189,9 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://example.com"}, meta={"proxy": "https://example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = None request.meta["proxy"] = None
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] is None assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -203,9 +201,9 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = None request.meta["proxy"] = None
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] is None assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -218,10 +216,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://example.com"}, meta={"proxy": "https://example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.com" request.meta["proxy"] = "https://user1:password1@example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user1", "user1",
@ -237,9 +235,9 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user2:password2@example.com" request.meta["proxy"] = "https://user2:password2@example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user2", "user2",
@ -260,10 +258,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.com" request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user1", "user1",
@ -273,7 +271,7 @@ class TestHttpProxyMiddleware:
request.meta["proxy"] = "https://example.com" request.meta["proxy"] = "https://example.com"
del request.headers[b"Proxy-Authorization"] del request.headers[b"Proxy-Authorization"]
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -283,10 +281,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://example.com"}, meta={"proxy": "https://example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.org" request.meta["proxy"] = "https://user1:password1@example.org"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org" assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user1", "user1",
@ -300,10 +298,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user1:password1@example.org" request.meta["proxy"] = "https://user1:password1@example.org"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org" assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user1", "user1",
@ -313,7 +311,7 @@ class TestHttpProxyMiddleware:
# Make sure, indirectly, that _auth_proxy is updated. # Make sure, indirectly, that _auth_proxy is updated.
request.meta["proxy"] = "https://example.com" request.meta["proxy"] = "https://example.com"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -323,10 +321,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://user2:password2@example.org" request.meta["proxy"] = "https://user2:password2@example.org"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.org" assert request.meta["proxy"] == "https://example.org"
encoded_credentials = middleware._basic_auth_header( encoded_credentials = middleware._basic_auth_header(
"user2", "user2",
@ -342,9 +340,9 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.org" request.meta["proxy"] = "https://example.org"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta == {"proxy": "https://example.org"} assert request.meta == {"proxy": "https://example.org"}
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -360,10 +358,10 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
request.meta["proxy"] = "https://example.org" request.meta["proxy"] = "https://example.org"
del request.headers[b"Proxy-Authorization"] del request.headers[b"Proxy-Authorization"]
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta == {"proxy": "https://example.org"} assert request.meta == {"proxy": "https://example.org"}
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -373,7 +371,7 @@ class TestHttpProxyMiddleware:
"https://example.com", "https://example.com",
headers={"Proxy-Authorization": "Basic foo"}, headers={"Proxy-Authorization": "Basic foo"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert "proxy" not in request.meta assert "proxy" not in request.meta
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -384,7 +382,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": "Basic foo"}, headers={"Proxy-Authorization": "Basic foo"},
meta={"proxy": None}, meta={"proxy": None},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] is None assert request.meta["proxy"] is None
assert b"Proxy-Authorization" not in request.headers assert b"Proxy-Authorization" not in request.headers
@ -398,16 +396,16 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": "Basic foo"}, headers={"Proxy-Authorization": "Basic foo"},
meta={"proxy": "https://example.com"}, meta={"proxy": "https://example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic foo" assert request.headers["Proxy-Authorization"] == b"Basic foo"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic foo" assert request.headers["Proxy-Authorization"] == b"Basic foo"
request.headers["Proxy-Authorization"] = b"Basic bar" request.headers["Proxy-Authorization"] = b"Basic bar"
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic bar" assert request.headers["Proxy-Authorization"] == b"Basic bar"
@ -422,7 +420,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": b"Basic " + encoded_credentials}, headers={"Proxy-Authorization": b"Basic " + encoded_credentials},
meta={"proxy": "https://user1:password1@example.com"}, meta={"proxy": "https://user1:password1@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials
@ -437,7 +435,7 @@ class TestHttpProxyMiddleware:
headers={"Proxy-Authorization": b"Basic " + encoded_credentials1}, headers={"Proxy-Authorization": b"Basic " + encoded_credentials1},
meta={"proxy": "https://user2:password2@example.com"}, meta={"proxy": "https://user2:password2@example.com"},
) )
assert middleware.process_request(request, spider) is None assert middleware.process_request(request) is None
assert request.meta["proxy"] == "https://example.com" assert request.meta["proxy"] == "https://example.com"
encoded_credentials2 = middleware._basic_auth_header( encoded_credentials2 = middleware._basic_auth_header(
"user2", "user2",

68
tests/test_downloadermiddleware_offsite.py
View File

@ -28,15 +28,15 @@ UNSET = object()
) )
def test_process_request_domain_filtering(allowed_domain, url, allowed): def test_process_request_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request(url) request = Request(url)
if allowed: if allowed:
assert mw.process_request(request, spider) is None assert mw.process_request(request) is None
else: else:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_request(request, spider) mw.process_request(request)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -50,18 +50,18 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
) )
def test_process_request_dont_filter(value, filtered): def test_process_request_dont_filter(value, filtered):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
kwargs = {} kwargs = {}
if value is not UNSET: if value is not UNSET:
kwargs["dont_filter"] = value kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs) request = Request("https://b.example", **kwargs)
if filtered: if filtered:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_request(request, spider) mw.process_request(request)
else: else:
assert mw.process_request(request, spider) is None assert mw.process_request(request) is None
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -79,9 +79,9 @@ def test_process_request_dont_filter(value, filtered):
) )
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
kwargs = {"meta": {}} kwargs = {"meta": {}}
if allow_offsite is not UNSET: if allow_offsite is not UNSET:
kwargs["meta"]["allow_offsite"] = allow_offsite kwargs["meta"]["allow_offsite"] = allow_offsite
@ -90,9 +90,9 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
request = Request("https://b.example", **kwargs) request = Request("https://b.example", **kwargs)
if filtered: if filtered:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_request(request, spider) mw.process_request(request)
else: else:
assert mw.process_request(request, spider) is None assert mw.process_request(request) is None
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -108,27 +108,27 @@ def test_process_request_no_allowed_domains(value):
kwargs = {} kwargs = {}
if value is not UNSET: if value is not UNSET:
kwargs["allowed_domains"] = value kwargs["allowed_domains"] = value
spider = crawler._create_spider(name="a", **kwargs) crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request("https://example.com") request = Request("https://example.com")
assert mw.process_request(request, spider) is None assert mw.process_request(request) is None
def test_process_request_invalid_domains(): def test_process_request_invalid_domains():
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning) warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request("https://a.example") request = Request("https://a.example")
assert mw.process_request(request, spider) is None assert mw.process_request(request) is None
for letter in ("b", "c"): for letter in ("b", "c"):
request = Request(f"https://{letter}.example") request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.process_request(request, spider) mw.process_request(request)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -149,15 +149,15 @@ def test_process_request_invalid_domains():
) )
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request(url) request = Request(url)
if allowed: if allowed:
assert mw.request_scheduled(request, spider) is None assert mw.request_scheduled(request, crawler.spider) is None
else: else:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider) mw.request_scheduled(request, crawler.spider)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -171,18 +171,18 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
) )
def test_request_scheduled_dont_filter(value, filtered): def test_request_scheduled_dont_filter(value, filtered):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
kwargs = {} kwargs = {}
if value is not UNSET: if value is not UNSET:
kwargs["dont_filter"] = value kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs) request = Request("https://b.example", **kwargs)
if filtered: if filtered:
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider) mw.request_scheduled(request, crawler.spider)
else: else:
assert mw.request_scheduled(request, spider) is None assert mw.request_scheduled(request, crawler.spider) is None
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -198,24 +198,24 @@ def test_request_scheduled_no_allowed_domains(value):
kwargs = {} kwargs = {}
if value is not UNSET: if value is not UNSET:
kwargs["allowed_domains"] = value kwargs["allowed_domains"] = value
spider = crawler._create_spider(name="a", **kwargs) crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request("https://example.com") request = Request("https://example.com")
assert mw.request_scheduled(request, spider) is None assert mw.request_scheduled(request, crawler.spider) is None
def test_request_scheduled_invalid_domains(): def test_request_scheduled_invalid_domains():
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler) mw = OffsiteMiddleware.from_crawler(crawler)
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning) warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider) mw.spider_opened(crawler.spider)
request = Request("https://a.example") request = Request("https://a.example")
assert mw.request_scheduled(request, spider) is None assert mw.request_scheduled(request, crawler.spider) is None
for letter in ("b", "c"): for letter in ("b", "c"):
request = Request(f"https://{letter}.example") request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, spider) mw.request_scheduled(request, crawler.spider)

283
tests/test_downloadermiddleware_redirect.py
View File

File diff suppressed because it is too large Load Diff

65
tests/test_downloadermiddleware_retry.py
View File

@ -19,20 +19,21 @@ from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.settings.default_settings import RETRY_EXCEPTIONS
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
class TestRetry: class TestRetry:
def setup_method(self): def setup_method(self):
self.crawler = get_crawler(Spider) self.crawler = get_crawler(DefaultSpider)
self.spider = self.crawler._create_spider("foo") self.crawler.spider = self.crawler._create_spider()
self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw.max_retry_times = 2 self.mw.max_retry_times = 2
def test_priority_adjust(self): def test_priority_adjust(self):
req = Request("http://www.scrapytest.org/503") req = Request("http://www.scrapytest.org/503")
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
req2 = self.mw.process_response(req, rsp, self.spider) req2 = self.mw.process_response(req, rsp)
assert req2.priority < req.priority assert req2.priority < req.priority
def test_404(self): def test_404(self):
@ -40,14 +41,14 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/404", body=b"", status=404) rsp = Response("http://www.scrapytest.org/404", body=b"", status=404)
# dont retry 404s # dont retry 404s
assert self.mw.process_response(req, rsp, self.spider) is rsp assert self.mw.process_response(req, rsp) is rsp
def test_dont_retry(self): def test_dont_retry(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry # first retry
r = self.mw.process_response(req, rsp, self.spider) r = self.mw.process_response(req, rsp)
assert r is rsp assert r is rsp
# Test retry when dont_retry set to False # Test retry when dont_retry set to False
@ -55,13 +56,13 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503")
# first retry # first retry
r = self.mw.process_response(req, rsp, self.spider) r = self.mw.process_response(req, rsp)
assert r is rsp assert r is rsp
def test_dont_retry_exc(self): def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
r = self.mw.process_exception(req, DNSLookupError(), self.spider) r = self.mw.process_exception(req, DNSLookupError())
assert r is None assert r is None
def test_503(self): def test_503(self):
@ -69,17 +70,17 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry # first retry
req = self.mw.process_response(req, rsp, self.spider) req = self.mw.process_response(req, rsp)
assert isinstance(req, Request) assert isinstance(req, Request)
assert req.meta["retry_times"] == 1 assert req.meta["retry_times"] == 1
# second retry # second retry
req = self.mw.process_response(req, rsp, self.spider) req = self.mw.process_response(req, rsp)
assert isinstance(req, Request) assert isinstance(req, Request)
assert req.meta["retry_times"] == 2 assert req.meta["retry_times"] == 2
# discard it # discard it
assert self.mw.process_response(req, rsp, self.spider) is rsp assert self.mw.process_response(req, rsp) is rsp
assert self.crawler.stats.get_value("retry/max_reached") == 1 assert self.crawler.stats.get_value("retry/max_reached") == 1
assert ( assert (
@ -118,7 +119,8 @@ class TestRetry:
settings_dict = { settings_dict = {
"RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc],
} }
crawler = get_crawler(Spider, settings_dict=settings_dict) crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler) mw = RetryMiddleware.from_crawler(crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}") req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw) self._test_retry_exception(req, exc("foo"), mw)
@ -128,65 +130,61 @@ class TestRetry:
mw = self.mw mw = self.mw
# first retry # first retry
req = mw.process_exception(req, exception, self.spider) req = mw.process_exception(req, exception)
assert isinstance(req, Request) assert isinstance(req, Request)
assert req.meta["retry_times"] == 1 assert req.meta["retry_times"] == 1
# second retry # second retry
req = mw.process_exception(req, exception, self.spider) req = mw.process_exception(req, exception)
assert isinstance(req, Request) assert isinstance(req, Request)
assert req.meta["retry_times"] == 2 assert req.meta["retry_times"] == 2
# discard it # discard it
req = mw.process_exception(req, exception, self.spider) req = mw.process_exception(req, exception)
assert req is None assert req is None
class TestMaxRetryTimes: class TestMaxRetryTimes:
invalid_url = "http://www.scrapytest.org/invalid_url" invalid_url = "http://www.scrapytest.org/invalid_url"
def get_spider_and_middleware(self, settings=None): def get_middleware(self, settings=None):
crawler = get_crawler(Spider, settings or {}) crawler = get_crawler(DefaultSpider, settings or {})
spider = crawler._create_spider("foo") crawler.spider = crawler._create_spider()
middleware = RetryMiddleware.from_crawler(crawler) return RetryMiddleware.from_crawler(crawler)
return spider, middleware
def test_with_settings_zero(self): def test_with_settings_zero(self):
max_retry_times = 0 max_retry_times = 0
settings = {"RETRY_TIMES": max_retry_times} settings = {"RETRY_TIMES": max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings) middleware = self.get_middleware(settings)
req = Request(self.invalid_url) req = Request(self.invalid_url)
self._test_retry( self._test_retry(
req, req,
DNSLookupError("foo"), DNSLookupError("foo"),
max_retry_times, max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
def test_with_metakey_zero(self): def test_with_metakey_zero(self):
max_retry_times = 0 max_retry_times = 0
spider, middleware = self.get_spider_and_middleware() middleware = self.get_middleware()
meta = {"max_retry_times": max_retry_times} meta = {"max_retry_times": max_retry_times}
req = Request(self.invalid_url, meta=meta) req = Request(self.invalid_url, meta=meta)
self._test_retry( self._test_retry(
req, req,
DNSLookupError("foo"), DNSLookupError("foo"),
max_retry_times, max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
def test_without_metakey(self): def test_without_metakey(self):
max_retry_times = 5 max_retry_times = 5
settings = {"RETRY_TIMES": max_retry_times} settings = {"RETRY_TIMES": max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings) middleware = self.get_middleware(settings)
req = Request(self.invalid_url) req = Request(self.invalid_url)
self._test_retry( self._test_retry(
req, req,
DNSLookupError("foo"), DNSLookupError("foo"),
max_retry_times, max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
@ -198,20 +196,18 @@ class TestMaxRetryTimes:
req2 = Request(self.invalid_url) req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times} settings = {"RETRY_TIMES": middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings) middleware = self.get_middleware(settings)
self._test_retry( self._test_retry(
req1, req1,
DNSLookupError("foo"), DNSLookupError("foo"),
meta_max_retry_times, meta_max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
self._test_retry( self._test_retry(
req2, req2,
DNSLookupError("foo"), DNSLookupError("foo"),
middleware_max_retry_times, middleware_max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
@ -223,26 +219,24 @@ class TestMaxRetryTimes:
req2 = Request(self.invalid_url) req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times} settings = {"RETRY_TIMES": middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings) middleware = self.get_middleware(settings)
self._test_retry( self._test_retry(
req1, req1,
DNSLookupError("foo"), DNSLookupError("foo"),
meta_max_retry_times, meta_max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
self._test_retry( self._test_retry(
req2, req2,
DNSLookupError("foo"), DNSLookupError("foo"),
middleware_max_retry_times, middleware_max_retry_times,
spider=spider,
middleware=middleware, middleware=middleware,
) )
def test_with_dont_retry(self): def test_with_dont_retry(self):
max_retry_times = 4 max_retry_times = 4
spider, middleware = self.get_spider_and_middleware() middleware = self.get_middleware()
meta = { meta = {
"max_retry_times": max_retry_times, "max_retry_times": max_retry_times,
"dont_retry": True, "dont_retry": True,
@ -252,7 +246,6 @@ class TestMaxRetryTimes:
req, req,
DNSLookupError("foo"), DNSLookupError("foo"),
0, 0,
spider=spider,
middleware=middleware, middleware=middleware,
) )
@ -261,18 +254,16 @@ class TestMaxRetryTimes:
req, req,
exception, exception,
max_retry_times, max_retry_times,
spider=None,
middleware=None, middleware=None,
): ):
spider = spider or self.spider
middleware = middleware or self.mw middleware = middleware or self.mw
for i in range(max_retry_times): for i in range(max_retry_times):
req = middleware.process_exception(req, exception, spider) req = middleware.process_exception(req, exception)
assert isinstance(req, Request) assert isinstance(req, Request)
# discard it # discard it
req = middleware.process_exception(req, exception, spider) req = middleware.process_exception(req, exception)
assert req is None assert req is None

43
tests/test_downloadermiddleware_robotstxt.py
View File

@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure from twisted.python import failure
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger
from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse from scrapy.http import Request, Response, TextResponse
from scrapy.http.request import NO_CALLBACK from scrapy.http.request import NO_CALLBACK
@ -83,10 +82,10 @@ Disallow: /some/randome/page.html
async def test_robotstxt_multiple_reqs(self) -> None: async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
d1 = deferred_from_coro( d1 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type] middleware.process_request(Request("http://site.local/allowed1"))
) )
d2 = deferred_from_coro( d2 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type] middleware.process_request(Request("http://site.local/allowed2"))
) )
await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True))
@ -94,8 +93,8 @@ Disallow: /some/randome/page.html
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_robotstxt_multiple_reqs_asyncio(self) -> None: async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
c1 = middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type] c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type] c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2) await asyncio.gather(c1, c2)
@deferred_f_from_coro_f @deferred_f_from_coro_f
@ -164,7 +163,7 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/static/"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_robotstxt_error(self): async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None:
self.crawler.settings.set("ROBOTSTXT_OBEY", True) self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = error.DNSLookupError("Robotstxt address not found") err = error.DNSLookupError("Robotstxt address not found")
@ -176,9 +175,8 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = return_failure self.crawler.engine.download_async.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler) middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) await middleware.process_request(Request("http://site.local"))
await middleware.process_request(Request("http://site.local"), None) assert "DNS lookup failed: Robotstxt address not found" in caplog.text
assert middleware._logerror.called
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_robotstxt_immediate_error(self): async def test_robotstxt_immediate_error(self):
@ -205,10 +203,13 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = ignore_request self.crawler.engine.download_async.side_effect = ignore_request
middleware = RobotsTxtMiddleware(self.crawler) middleware = RobotsTxtMiddleware(self.crawler)
mw_module_logger.error = mock.MagicMock() with mock.patch(
"scrapy.downloadermiddlewares.robotstxt.logger"
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) ) as mw_module_logger:
assert not mw_module_logger.error.called # type: ignore[attr-defined] await self.assertNotIgnored(
Request("http://site.local/allowed"), middleware
)
assert not mw_module_logger.error.called
def test_robotstxt_user_agent_setting(self): def test_robotstxt_user_agent_setting(self):
crawler = self._get_successful_crawler() crawler = self._get_successful_crawler()
@ -216,7 +217,7 @@ Disallow: /some/randome/page.html
crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)") crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")
middleware = RobotsTxtMiddleware(crawler) middleware = RobotsTxtMiddleware(crawler)
rp = mock.MagicMock(return_value=True) rp = mock.MagicMock(return_value=True)
middleware.process_request_2(rp, Request("http://site.local/allowed"), None) middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@deferred_f_from_coro_f @deferred_f_from_coro_f
@ -224,34 +225,30 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock() middleware.process_request_2 = mock.MagicMock()
await middleware.process_request( await middleware.process_request(Request("data:text/plain,Hello World data"))
Request("data:text/plain,Hello World data"), None
)
assert not middleware.process_request_2.called assert not middleware.process_request_2.called
await middleware.process_request( await middleware.process_request(
Request("file:///tests/sample_data/test_site/nothinghere.html"), None Request("file:///tests/sample_data/test_site/nothinghere.html")
) )
assert not middleware.process_request_2.called assert not middleware.process_request_2.called
await middleware.process_request(Request("http://site.local/allowed"), None) await middleware.process_request(Request("http://site.local/allowed"))
assert middleware.process_request_2.called assert middleware.process_request_2.called
async def assertNotIgnored( async def assertNotIgnored(
self, request: Request, middleware: RobotsTxtMiddleware self, request: Request, middleware: RobotsTxtMiddleware
) -> None: ) -> None:
spider = None # not actually used
try: try:
await middleware.process_request(request, spider) # type: ignore[arg-type] await middleware.process_request(request)
except IgnoreRequest: except IgnoreRequest:
pytest.fail("IgnoreRequest was raised unexpectedly") pytest.fail("IgnoreRequest was raised unexpectedly")
async def assertIgnored( async def assertIgnored(
self, request: Request, middleware: RobotsTxtMiddleware self, request: Request, middleware: RobotsTxtMiddleware
) -> None: ) -> None:
spider = None # not actually used
with pytest.raises(IgnoreRequest): with pytest.raises(IgnoreRequest):
await middleware.process_request(request, spider) # type: ignore[arg-type] await middleware.process_request(request)
def assertRobotsTxtRequested(self, base_url: str) -> None: def assertRobotsTxtRequested(self, base_url: str) -> None:
calls = self.crawler.engine.download_async.call_args_list calls = self.crawler.engine.download_async.call_args_list

7
tests/test_downloadermiddleware_stats.py
View File

@ -11,7 +11,6 @@ class MyException(Exception):
class TestDownloaderStats: class TestDownloaderStats:
def setup_method(self): def setup_method(self):
self.crawler = get_crawler(Spider) self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = DownloaderStats(self.crawler.stats) self.mw = DownloaderStats(self.crawler.stats)
self.crawler.stats.open_spider() self.crawler.stats.open_spider()
@ -25,15 +24,15 @@ class TestDownloaderStats:
) )
def test_process_request(self): def test_process_request(self):
self.mw.process_request(self.req, self.spider) self.mw.process_request(self.req)
self.assertStatsEqual("downloader/request_count", 1) self.assertStatsEqual("downloader/request_count", 1)
def test_process_response(self): def test_process_response(self):
self.mw.process_response(self.req, self.res, self.spider) self.mw.process_response(self.req, self.res)
self.assertStatsEqual("downloader/response_count", 1) self.assertStatsEqual("downloader/response_count", 1)
def test_process_exception(self): def test_process_exception(self):
self.mw.process_exception(self.req, MyException(), self.spider) self.mw.process_exception(self.req, MyException())
self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual("downloader/exception_count", 1)
self.assertStatsEqual( self.assertStatsEqual(
"downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException", "downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException",

12
tests/test_downloadermiddleware_useragent.py
View File

@ -11,9 +11,9 @@ class TestUserAgentMiddleware:
return spider, UserAgentMiddleware.from_crawler(crawler) return spider, UserAgentMiddleware.from_crawler(crawler)
def test_default_agent(self): def test_default_agent(self):
spider, mw = self.get_spider_and_mw("default_useragent") _, mw = self.get_spider_and_mw("default_useragent")
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"default_useragent" assert req.headers["User-Agent"] == b"default_useragent"
def test_remove_agent(self): def test_remove_agent(self):
@ -22,7 +22,7 @@ class TestUserAgentMiddleware:
spider.user_agent = None spider.user_agent = None
mw.spider_opened(spider) mw.spider_opened(spider)
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.headers.get("User-Agent") is None assert req.headers.get("User-Agent") is None
def test_spider_agent(self): def test_spider_agent(self):
@ -30,7 +30,7 @@ class TestUserAgentMiddleware:
spider.user_agent = "spider_useragent" spider.user_agent = "spider_useragent"
mw.spider_opened(spider) mw.spider_opened(spider)
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"spider_useragent" assert req.headers["User-Agent"] == b"spider_useragent"
def test_header_agent(self): def test_header_agent(self):
@ -40,7 +40,7 @@ class TestUserAgentMiddleware:
req = Request( req = Request(
"http://scrapytest.org/", headers={"User-Agent": "header_useragent"} "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
) )
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert req.headers["User-Agent"] == b"header_useragent" assert req.headers["User-Agent"] == b"header_useragent"
def test_no_agent(self): def test_no_agent(self):
@ -48,5 +48,5 @@ class TestUserAgentMiddleware:
spider.user_agent = None spider.user_agent = None
mw.spider_opened(spider) mw.spider_opened(spider)
req = Request("http://scrapytest.org/") req = Request("http://scrapytest.org/")
assert mw.process_request(req, spider) is None assert mw.process_request(req) is None
assert "User-Agent" not in req.headers assert "User-Agent" not in req.headers

4
tests/test_middleware.py
View File

@ -20,7 +20,7 @@ class M1:
def close_spider(self, spider): def close_spider(self, spider):
pass pass
def process(self, response, request, spider): def process(self, response, request):
pass pass
@ -33,7 +33,7 @@ class M2:
class M3: class M3:
def process(self, response, request, spider): def process(self, response, request):
pass pass

19
tests/test_request_attribute_binding.py
View File

@ -11,18 +11,18 @@ OVERRIDDEN_URL = "https://example.org"
class ProcessResponseMiddleware: class ProcessResponseMiddleware:
def process_response(self, request, response, spider): def process_response(self, request, response):
return response.replace(request=Request(OVERRIDDEN_URL)) return response.replace(request=Request(OVERRIDDEN_URL))
class RaiseExceptionRequestMiddleware: class RaiseExceptionRequestMiddleware:
def process_request(self, request, spider): def process_request(self, request):
1 / 0 1 / 0
return request return request
class CatchExceptionOverrideRequestMiddleware: class CatchExceptionOverrideRequestMiddleware:
def process_exception(self, request, exception, spider): def process_exception(self, request, exception):
return Response( return Response(
url="http://localhost/", url="http://localhost/",
body=b"Caught " + exception.__class__.__name__.encode("utf-8"), body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
@ -31,7 +31,7 @@ class CatchExceptionOverrideRequestMiddleware:
class CatchExceptionDoNotOverrideRequestMiddleware: class CatchExceptionDoNotOverrideRequestMiddleware:
def process_exception(self, request, exception, spider): def process_exception(self, request, exception):
return Response( return Response(
url="http://localhost/", url="http://localhost/",
body=b"Caught " + exception.__class__.__name__.encode("utf-8"), body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
@ -46,10 +46,17 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
class AlternativeCallbacksMiddleware: class AlternativeCallbacksMiddleware:
def process_response(self, request, response, spider): def __init__(self, crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_response(self, request, response):
new_request = request.replace( new_request = request.replace(
url=OVERRIDDEN_URL, url=OVERRIDDEN_URL,
callback=spider.alt_callback, callback=self.crawler.spider.alt_callback,
cb_kwargs={"foo": "bar"}, cb_kwargs={"foo": "bar"},
) )
return response.replace(request=new_request) return response.replace(request=new_request)

4
tests/test_request_cb_kwargs.py
View File

@ -12,11 +12,11 @@ class InjectArgumentsDownloaderMiddleware:
Make sure downloader middlewares are able to update the keyword arguments Make sure downloader middlewares are able to update the keyword arguments
""" """
def process_request(self, request, spider): def process_request(self, request):
if request.callback.__name__ == "parse_downloader_mw": if request.callback.__name__ == "parse_downloader_mw":
request.cb_kwargs["from_process_request"] = True request.cb_kwargs["from_process_request"] = True
def process_response(self, request, response, spider): def process_response(self, request, response):
if request.callback.__name__ == "parse_downloader_mw": if request.callback.__name__ == "parse_downloader_mw":
request.cb_kwargs["from_process_response"] = True request.cb_kwargs["from_process_response"] = True
return response return response

31
tests/test_spidermiddleware_referer.py
View File

@ -31,15 +31,13 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy, StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy, UnsafeUrlPolicy,
) )
from scrapy.spiders import Spider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Sequence from collections.abc import Sequence
from scrapy.crawler import Crawler
@pytest.fixture
def spider() -> Spider:
return Spider("foo")
class TestRefererMiddleware: class TestRefererMiddleware:
@ -1011,20 +1009,24 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
] ]
@pytest.fixture @pytest.fixture
def referrermw(self) -> RefererMiddleware: def crawler(self) -> Crawler:
settings = Settings(self.settings) crawler = get_crawler(DefaultSpider, self.settings)
return RefererMiddleware(settings) crawler.spider = crawler._create_spider()
return crawler
@pytest.fixture @pytest.fixture
def redirectmw(self) -> RedirectMiddleware: def referrermw(self, crawler: Crawler) -> RefererMiddleware:
settings = Settings(self.settings) return RefererMiddleware.from_crawler(crawler)
return RedirectMiddleware(settings)
@pytest.fixture
def redirectmw(self, crawler: Crawler) -> RedirectMiddleware:
return RedirectMiddleware.from_crawler(crawler)
def test( # type: ignore[override] def test( # type: ignore[override]
self, self,
crawler: Crawler,
referrermw: RefererMiddleware, referrermw: RefererMiddleware,
redirectmw: RedirectMiddleware, redirectmw: RedirectMiddleware,
spider: Spider,
) -> None: ) -> None:
for ( for (
parent, parent,
@ -1044,9 +1046,10 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
request.url, headers={"Location": url}, status=status request.url, headers={"Location": url}, status=status
) )
request = cast( request = cast(
"Request", redirectmw.process_response(request, response, spider) "Request", redirectmw.process_response(request, response)
) )
referrermw.request_scheduled(request, spider) assert crawler.spider
referrermw.request_scheduled(request, crawler.spider)
assert isinstance(request, Request) assert isinstance(request, Request)
assert request.headers.get("Referer") == final_referrer assert request.headers.get("Referer") == final_referrer