diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index a0ca73978..e679f9626 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -87,7 +87,7 @@ class RFPDupeFilter(BaseDupeFilter): fp = self.request_fingerprint(request) if fp in self.fingerprints: redirect_fps = request.meta.get("redirect_fingerprints", set()) - return fp not in redirect_fps + return fp not in {_fp.hex() for _fp in redirect_fps} self.fingerprints.add(fp) if self.file: self.file.write(fp + "\n") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index dc15b672c..582b56a7c 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -9,6 +9,8 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from .test_dupefilters import _get_dupefilter + class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): @@ -247,6 +249,96 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_self_redirect_direct(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + self.assertFalse(dupefilter.request_seen(request1)) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/a"}, + ) + request2 = self.mw.process_response(request1, response1, self.spider) + self.assertIsInstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + self.assertEqual(fingerprint1, fingerprint2) + + self.assertFalse(dupefilter.request_seen(request2)) + + def test_self_redirect_indirect(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + self.assertFalse(dupefilter.request_seen(request1)) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/b"}, + ) + request2 = self.mw.process_response(request1, response1, self.spider) + self.assertIsInstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + self.assertNotEqual(fingerprint1, fingerprint2) + + self.assertFalse(dupefilter.request_seen(request2)) + + response2 = Response( + request2.url, + status=302, + headers={"Location": "/a"}, + ) + request3 = self.mw.process_response(request2, response2, self.spider) + self.assertIsInstance(request3, Request) + fingerprint3 = self.crawler.request_fingerprinter.fingerprint(request3) + self.assertEqual(fingerprint1, fingerprint3) + + self.assertFalse(dupefilter.request_seen(request3)) + + def test_self_redirect_zigzag(self): + dupefilter = _get_dupefilter(crawler=self.crawler) + request1 = Request("https://example.com/a") + self.assertFalse(dupefilter.request_seen(request1)) + + response1 = Response( + request1.url, + status=302, + headers={"Location": "/b"}, + ) + request2 = self.mw.process_response(request1, response1, self.spider) + self.assertIsInstance(request2, Request) + fingerprint1 = self.crawler.request_fingerprinter.fingerprint(request1) + fingerprint2 = self.crawler.request_fingerprinter.fingerprint(request2) + self.assertNotEqual(fingerprint1, fingerprint2) + + self.assertFalse(dupefilter.request_seen(request2)) + + response2 = Response( + request2.url, + status=302, + headers={"Location": "/a"}, + ) + request3 = self.mw.process_response(request2, response2, self.spider) + self.assertIsInstance(request3, Request) + fingerprint3 = self.crawler.request_fingerprinter.fingerprint(request3) + self.assertEqual(fingerprint1, fingerprint3) + + self.assertFalse(dupefilter.request_seen(request3)) + + response3 = Response( + request3.url, + status=302, + headers={"Location": "/b"}, + ) + request4 = self.mw.process_response(request3, response3, self.spider) + self.assertIsInstance(request4, Request) + fingerprint4 = self.crawler.request_fingerprinter.fingerprint(request4) + self.assertEqual(fingerprint2, fingerprint4) + + self.assertFalse(dupefilter.request_seen(request4)) + class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self):