Added request_fingerprint method to dupefilter classes so they could be easily subclassed without need to override entire request_seen method.

.gitignore

@@ -7,3 +7,4 @@ docs/build
 venv
 build
 dist
+.idea

scrapy/dupefilter.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
 import os
-from scrapy.utils.request import request_fingerprint
-from scrapy.utils.job import job_dir
+
 from scrapy import log
+from scrapy.utils.job import job_dir
+from scrapy.utils.request import request_fingerprint
 
 
 class BaseDupeFilter(object):
@@ -14,15 +15,19 @@ class BaseDupeFilter(object):
     def request_seen(self, request):
         return False
 
+    def request_fingerprint(self, request):
+        pass
+
     def open(self):  # can return deferred
         pass
 
-    def close(self, reason): # can return a deferred
+    def close(self, reason):  # can return a deferred
         pass
 
-    def log(self, request, spider): # log that a request has been filtered
+    def log(self, request, spider):  # log that a request has been filtered
         pass
 
+
 class RFPDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
 
@@ -39,13 +44,19 @@ class RFPDupeFilter(BaseDupeFilter):
         return cls(job_dir(settings))
 
     def request_seen(self, request):
-        fp = request_fingerprint(request)
+        fp = self.request_fingerprint(request)
         if fp in self.fingerprints:
             return True
         self.fingerprints.add(fp)
         if self.file:
             self.file.write(fp + os.linesep)
 
+    def request_fingerprint(self, request):
+        """Override this method to implement a different duplicate check.
+
+        """
+        return request_fingerprint(request)
+
     def close(self, reason):
         if self.file:
             self.file.close()