RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve how request_seen handles redirect fingerprints

This commit is contained in:
Adrián Chaves 2020-02-06 18:57:07 +01:00
parent fe5e412114
commit 03311f8857
1 changed files with 2 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/dupefilters.py
View File

@ -45,12 +45,9 @@ class RFPDupeFilter(BaseDupeFilter):
def request_seen(self, request):
fp = self.request_fingerprint(request)
redirect_fps = request.meta.get('redirect_fingerprints', set())
if fp in redirect_fps:
assert fp in self.fingerprints
return False
if fp in self.fingerprints:
return True
redirect_fps = request.meta.get('redirect_fingerprints', set())
return fp not in redirect_fps
self.fingerprints.add(fp)
if self.file:
self.file.write(fp + os.linesep)