diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 77cb5aa94..c910d3e0d 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy.http import HtmlResponse +from scrapy.utils.request import request_fingerprint from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -37,6 +38,9 @@ class BaseRedirectMiddleware(object): [request.url] redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ [reason] + fingerprints = request.meta.get('redirect_fingerprints', set()) + fingerprint = request_fingerprint(request) + redirected.meta['redirect_fingerprints'] = fingerprints | {fingerprint} redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ea6a4cfc3..0b2986caa 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -45,6 +45,10 @@ class RFPDupeFilter(BaseDupeFilter): def request_seen(self, request): fp = self.request_fingerprint(request) + redirect_fps = request.meta.get('redirect_fingerprints', set()) + if fp in redirect_fps: + assert fp in self.fingerprints + return False if fp in self.fingerprints: return True self.fingerprints.add(fp)