diff --git a/.gitignore b/.gitignore index 564145da5..4eb80012f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ docs/build venv build dist +.idea diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py index 67989d8d3..7e4bf3153 100644 --- a/scrapy/dupefilter.py +++ b/scrapy/dupefilter.py @@ -1,8 +1,9 @@ from __future__ import print_function import os -from scrapy.utils.request import request_fingerprint -from scrapy.utils.job import job_dir + from scrapy import log +from scrapy.utils.job import job_dir +from scrapy.utils.request import request_fingerprint class BaseDupeFilter(object): @@ -14,15 +15,19 @@ class BaseDupeFilter(object): def request_seen(self, request): return False + def request_fingerprint(self, request): + pass + def open(self): # can return deferred pass - def close(self, reason): # can return a deferred + def close(self, reason): # can return a deferred pass - def log(self, request, spider): # log that a request has been filtered + def log(self, request, spider): # log that a request has been filtered pass + class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" @@ -39,13 +44,19 @@ class RFPDupeFilter(BaseDupeFilter): return cls(job_dir(settings)) def request_seen(self, request): - fp = request_fingerprint(request) + fp = self.request_fingerprint(request) if fp in self.fingerprints: return True self.fingerprints.add(fp) if self.file: self.file.write(fp + os.linesep) + def request_fingerprint(self, request): + """Override this method to implement a different duplicate check. + + """ + return request_fingerprint(request) + def close(self, reason): if self.file: self.file.close()