From 886131c7b2f2e792fc139e5660f908239836388c Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 12 Mar 2026 13:54:00 +0100 Subject: [PATCH] Run pre-commit --- scrapy/downloadermiddlewares/redirect.py | 18 ++++----- tests/test_downloadermiddleware_redirect.py | 41 ++++++++++++++------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 9520a5cbb..69c58ca49 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -61,12 +61,12 @@ class BaseRedirectMiddleware: request.headers.pop("Referer", None) if not self._referer_spider_middleware: return - self._referer_spider_middleware.get_processed_request( - request, response - ) + self._referer_spider_middleware.get_processed_request(request, response) def _engine_started(self) -> None: - self._referer_spider_middleware = self.crawler.get_spider_middleware(RefererMiddleware) + self._referer_spider_middleware = self.crawler.get_spider_middleware( + RefererMiddleware + ) if self._referer_spider_middleware: return redirect_cls = global_object_name(self.__class__) @@ -120,7 +120,6 @@ class BaseRedirectMiddleware: ) raise IgnoreRequest("max redirections reached") - def _build_redirect_request( self, source_request: Request, response: Response, *, url: str, **kwargs: Any ) -> Request: @@ -236,11 +235,12 @@ class RedirectMiddleware(BaseRedirectMiddleware): if urlparse_cached(redirected).scheme not in {"http", "https"}: return response - if ( - (response.status in (301, 302) and request.method == "POST") - or (response.status == 303 and request.method not in ("GET", "HEAD")) + if (response.status in (301, 302) and request.method == "POST") or ( + response.status == 303 and request.method not in ("GET", "HEAD") ): - redirected = self._redirect_request_using_get(request, response, redirected_url) + redirected = self._redirect_request_using_get( + request, response, redirected_url + ) return self._redirect(redirected, request, response.status) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4197c7c86..89185aafb 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1197,7 +1197,9 @@ class TestRedirectMiddleware(Base.Test): def test_redirect_strips_referer_no_middleware(self): source_url = "http://www.example.com/302" redirect_url = "http://www.example.com/redirected" - source_request = Request(source_url, headers={"Referer": "http://example.com/old"}) + source_request = Request( + source_url, headers={"Referer": "http://example.com/old"} + ) response = Response(source_url, headers={"Location": redirect_url}, status=302) redirect_mw = self.mwcls.from_crawler(get_crawler()) redirect_mw._referer_spider_middleware = None @@ -1213,9 +1215,14 @@ class TestRedirectMiddleware(Base.Test): source_url, method="POST", body=body, - headers={"Content-Type": "application/json", "Content-Length": str(len(body))}, + headers={ + "Content-Type": "application/json", + "Content-Length": str(len(body)), + }, + ) + response1 = Response( + source_url, headers={"Location": target_url}, status=status ) - response1 = Response(source_url, headers={"Location": target_url}, status=status) redirect_request = self.mw.process_response(request, response1) assert isinstance(redirect_request, Request) assert redirect_request.url == target_url @@ -1480,6 +1487,7 @@ def test_meta_refresh_schemes(url, location, target): else: assert isinstance(redirect, Request) + @pytest.mark.parametrize( ("policy", "source_url", "target_url", "expected_referrer"), [ @@ -1507,9 +1515,7 @@ def test_meta_refresh_schemes(url, location, target): ), ], ) -def test_response_referrer_policy( - policy, source_url, target_url, expected_referrer -): +def test_response_referrer_policy(policy, source_url, target_url, expected_referrer): crawler = get_crawler() referrer_mw = build_from_crawler(RefererMiddleware, crawler) redirect_mw = build_from_crawler(RedirectMiddleware, crawler) @@ -1519,11 +1525,11 @@ def test_response_referrer_policy( if policy: extra_headers["Referrer-Policy"] = policy response_redirect = Response( - source_request.url, status=301, headers={"Location": target_url, **extra_headers} - ) - source_request = redirect_mw.process_response( - source_request, response_redirect + source_request.url, + status=301, + headers={"Location": target_url, **extra_headers}, ) + source_request = redirect_mw.process_response(source_request, response_redirect) assert isinstance(source_request, Request) assert source_request.headers.get("Referer") == expected_referrer @@ -1547,7 +1553,10 @@ def test_warning_redirect_middleware(caplog): "scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no " "scrapy.spidermiddlewares.referer.RefererMiddleware" ) in caplog.text - assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) assert ( "replace scrapy.downloadermiddlewares.redirect.RedirectMiddleware " "with a subclass that overrides the handle_referer() method" @@ -1564,7 +1573,10 @@ def test_warning_meta_refresh_middleware(caplog): "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no " "scrapy.spidermiddlewares.referer.RefererMiddleware" ) in caplog.text - assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) assert ( "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " "with a subclass that overrides the handle_referer() method" @@ -1584,7 +1596,10 @@ def test_warning_subclass(caplog): "test_warning_subclass..MyRedirectMiddleware found no " "scrapy.spidermiddlewares.referer.RefererMiddleware" ) in caplog.text - assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) assert "edit " in caplog.text assert "test_warning_subclass..MyRedirectMiddleware" in caplog.text assert (