From 03311f8857ec94399aff56b153be40b676b3d310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 6 Feb 2020 18:57:07 +0100 Subject: [PATCH] Improve how request_seen handles redirect fingerprints --- scrapy/dupefilters.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0b2986caa..ae3eafc24 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -45,12 +45,9 @@ class RFPDupeFilter(BaseDupeFilter): def request_seen(self, request): fp = self.request_fingerprint(request) - redirect_fps = request.meta.get('redirect_fingerprints', set()) - if fp in redirect_fps: - assert fp in self.fingerprints - return False if fp in self.fingerprints: - return True + redirect_fps = request.meta.get('redirect_fingerprints', set()) + return fp not in redirect_fps self.fingerprints.add(fp) if self.file: self.file.write(fp + os.linesep)