From fe5e41211483083644cbab166d6b9b97823a90d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 6 Feb 2020 18:48:03 +0100 Subject: [PATCH] Do not filter duplicate requests within a redirect chain --- scrapy/downloadermiddlewares/redirect.py | 4 ++++ scrapy/dupefilters.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 77cb5aa94..c910d3e0d 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy.http import HtmlResponse +from scrapy.utils.request import request_fingerprint from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -37,6 +38,9 @@ class BaseRedirectMiddleware(object): [request.url] redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ [reason] + fingerprints = request.meta.get('redirect_fingerprints', set()) + fingerprint = request_fingerprint(request) + redirected.meta['redirect_fingerprints'] = fingerprints | {fingerprint} redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ea6a4cfc3..0b2986caa 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -45,6 +45,10 @@ class RFPDupeFilter(BaseDupeFilter): def request_seen(self, request): fp = self.request_fingerprint(request) + redirect_fps = request.meta.get('redirect_fingerprints', set()) + if fp in redirect_fps: + assert fp in self.fingerprints + return False if fp in self.fingerprints: return True self.fingerprints.add(fp)