From 296db1dc09de0d6909e27f3450da9948ec2798a1 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 12 Mar 2013 19:28:43 -0300
Subject: [PATCH] log (just once) when duplicate requests are filtered out.
 closes #105, #249

---
 scrapy/core/scheduler.py |  1 +
 scrapy/dupefilter.py     | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 3d8031557..ef661992f 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -46,6 +46,7 @@ class Scheduler(object):
 
     def enqueue_request(self, request):
         if not request.dont_filter and self.df.request_seen(request):
+            self.df.log(request, self.spider)
             return
         dqok = self._dqpush(request)
         if dqok:
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
index 94dc88ba9..4e3cb1cf2 100644
--- a/scrapy/dupefilter.py
+++ b/scrapy/dupefilter.py
@@ -1,6 +1,7 @@
 import os
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.job import job_dir
+from scrapy import log
 
 
 class BaseDupeFilter(object):
@@ -18,6 +19,8 @@ class BaseDupeFilter(object):
     def close(self, reason): # can return a deferred
         pass
 
+    def log(self, request, spider): # log that a request has been filtered
+        pass
 
 class RFPDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
@@ -25,6 +28,7 @@ class RFPDupeFilter(BaseDupeFilter):
     def __init__(self, path=None):
         self.file = None
         self.fingerprints = set()
+        self.logdupes = True
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.fingerprints.update(x.rstrip() for x in self.file)
@@ -44,3 +48,9 @@ class RFPDupeFilter(BaseDupeFilter):
     def close(self, reason):
         if self.file:
             self.file.close()
+
+    def log(self, request, spider):
+        if self.logdupes:
+            fmt = "Filtered duplicate request: %(request)s - no more duplicates will be shown (see DUPEFILTER_CLASS)"
+            log.msg(format=fmt, request=request, level=log.DEBUG, spider=spider)
+            self.logdupes = False