diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 45af5c67a..c2f296b4e 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -28,8 +28,8 @@ logger = logging.getLogger(__name__) class BaseRedirectMiddleware: - crawler: Crawler enabled_setting: str = "REDIRECT_ENABLED" + crawler: Crawler def __init__(self, settings: BaseSettings): if not settings.getbool(self.enabled_setting): @@ -105,6 +105,10 @@ class BaseRedirectMiddleware: *request.meta.get("redirect_reasons", []), reason, ] + assert self.crawler.request_fingerprinter is not None + redirected.meta["redirect_fingerprints"] = request.meta.get( + "redirect_fingerprints", set() + ) | {self.crawler.request_fingerprinter.fingerprint(request)} redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 09f0be63b..73a129fac 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -105,7 +105,8 @@ class RFPDupeFilter(BaseDupeFilter): def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: - return True + redirect_fps = request.meta.get("redirect_fingerprints", set()) + return fp not in {_fp.hex() for _fp in redirect_fps} self.fingerprints.add(fp) if self.file: self.file.write(fp + "\n") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4cebcb281..de5200cbd 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -19,15 +19,17 @@ from scrapy.utils.test import get_crawler from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS +from .test_dupefilters import _get_dupefilter + class TestRedirectMiddleware(TestRedirectBase): mwcls = RedirectMiddleware reason = 302 def setup_method(self): - crawler = get_crawler(DefaultSpider) - crawler.spider = crawler._create_spider() - self.mw = build_from_crawler(self.mwcls, crawler) + self.crawler = get_crawler(DefaultSpider) + self.crawler.spider = self.crawler._create_spider() + self.mw = build_from_crawler(self.mwcls, self.crawler) def get_response(self, request, location, status=302): headers = {"Location": location} @@ -348,6 +350,96 @@ class TestRedirectMiddleware(TestRedirectBase): response = Response(request.url, status=302) assert self.mw.process_response(request, response) is response + def test_self_redirect_direct(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + assert not dupefilter.request_seen(request1) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/a"}, + ) + request2 = self.mw.process_response(request1, response1) + assert isinstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + assert fingerprint1 == fingerprint2 + + assert not dupefilter.request_seen(request2) + + def test_self_redirect_indirect(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + assert not dupefilter.request_seen(request1) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/b"}, + ) + request2 = self.mw.process_response(request1, response1) + assert isinstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + assert fingerprint1 != fingerprint2 + + assert not dupefilter.request_seen(request2) + + response2 = Response( + request2.url, + status=302, + headers={"Location": "/a"}, + ) + request3 = self.mw.process_response(request2, response2) + assert isinstance(request3, Request) + fingerprint3 = self.crawler.request_fingerprinter.fingerprint(request3) + assert fingerprint1 == fingerprint3 + + assert not dupefilter.request_seen(request3) + + def test_self_redirect_zigzag(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + assert not dupefilter.request_seen(request1) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/b"}, + ) + request2 = self.mw.process_response(request1, response1) + assert isinstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + assert fingerprint1 != fingerprint2 + + assert not dupefilter.request_seen(request2) + + response2 = Response( + request2.url, + status=302, + headers={"Location": "/a"}, + ) + request3 = self.mw.process_response(request2, response2) + assert isinstance(request3, Request) + fingerprint3 = self.crawler.request_fingerprinter.fingerprint(request3) + assert fingerprint1 == fingerprint3 + + assert not dupefilter.request_seen(request3) + + response3 = Response( + request3.url, + status=302, + headers={"Location": "/b"}, + ) + request4 = self.mw.process_response(request3, response3) + assert isinstance(request4, Request) + fingerprint4 = self.crawler.request_fingerprinter.fingerprint(request4) + assert fingerprint2 == fingerprint4 + + assert not dupefilter.request_seen(request4) + @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target):