From 6e4c77c684b078ed9ac96b14f2f6d143b8403608 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Sat, 15 Feb 2014 17:47:07 +0200
Subject: [PATCH 1/3] test for RFPDupeFilter.request_fingerprint overriding

---
 scrapy/tests/test_dupefilter.py | 48 +++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py
index 5ee73d499..b0dd9546b 100644
--- a/scrapy/tests/test_dupefilter.py
+++ b/scrapy/tests/test_dupefilter.py
@@ -1,23 +1,55 @@
+import hashlib
 import unittest
 
-from scrapy.http import Request
 from scrapy.dupefilter import RFPDupeFilter
+from scrapy.http import Request
 
 
 class RFPDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
-        filter = RFPDupeFilter()
-        filter.open()
+        dupefilter = RFPDupeFilter()
+        dupefilter.open()
 
         r1 = Request('http://scrapytest.org/1')
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
 
-        assert not filter.request_seen(r1)
-        assert filter.request_seen(r1)
+        assert not dupefilter.request_seen(r1)
+        assert dupefilter.request_seen(r1)
 
-        assert not filter.request_seen(r2)
-        assert filter.request_seen(r3)
+        assert not dupefilter.request_seen(r2)
+        assert dupefilter.request_seen(r3)
 
-        filter.close('finished')
+        dupefilter.close('finished')
+
+    def test_request_fingerprint(self):
+        """Test if customization of request_fingerprint method will change
+        output of request_seen.
+
+        """
+        r1 = Request('http://scrapytest.org/index.html')
+        r2 = Request('http://scrapytest.org/INDEX.html')
+
+        dupefilter = RFPDupeFilter()
+        dupefilter.open()
+
+        assert not dupefilter.request_seen(r1)
+        assert not dupefilter.request_seen(r2)
+
+        dupefilter.close('finished')
+
+        class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+
+            def request_fingerprint(self, request):
+                fp = hashlib.sha1()
+                fp.update(request.url.lower())
+                return fp.hexdigest()
+
+        case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
+        case_insensitive_dupefilter.open()
+
+        assert not case_insensitive_dupefilter.request_seen(r1)
+        assert case_insensitive_dupefilter.request_seen(r2)
+
+        case_insensitive_dupefilter.close('finished')

From baaa077725f4eab38227999a0eaf4d0b9df16f99 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Sat, 15 Feb 2014 17:48:32 +0200
Subject: [PATCH 2/3] added note about RFPDupeFilter.request_fingerprint
 overriding to the settings documentation

---
 docs/topics/settings.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 12dda4322..78728ac97 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -408,7 +408,9 @@ Default: ``'scrapy.dupefilter.RFPDupeFilter'``
 The class used to detect and filter duplicate requests.
 
 The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function.
+the ``scrapy.utils.request.request_fingerprint`` function. In order to change
+the way duplicates are checked you could subclass ``RFPDupeFilter`` and
+override its ``request_fingerprint`` method.
 
 .. setting:: DUPEFILTER_DEBUG
 

From 4aa6179af7158393662f13898253bf78849bf2d7 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Sat, 26 Apr 2014 15:46:53 +0300
Subject: [PATCH 3/3] added short RFPDupeFilter.request_fingerprint interface
 description

---
 docs/topics/settings.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 78728ac97..035e12c82 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -410,7 +410,9 @@ The class used to detect and filter duplicate requests.
 The default (``RFPDupeFilter``) filters based on request fingerprint using
 the ``scrapy.utils.request.request_fingerprint`` function. In order to change
 the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method.
+override its ``request_fingerprint`` method. This method should accept
+scrapy :class:`~scrapy.http.Request` object and return its fingerprint
+(a string).
 
 .. setting:: DUPEFILTER_DEBUG