duplicatefilter: filter request prior to reach spider

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40884

scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py

@@ -6,7 +6,7 @@ from pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.http import Request
-from scrapy.core.exceptions import NotConfigured
+from scrapy.core.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import load_object
 from scrapy.conf import settings
@@ -38,15 +38,17 @@ class DuplicatesFilterMiddleware(object):
         dispatcher.connect(self.filter.open, signals.domain_open)
         dispatcher.connect(self.filter.close, signals.domain_closed)
 
+    def process_spider_input(self, response, spider):
+        if not self.filter.add(spider.domain_name, response.request):
+            raise IgnoreRequest("Skipped (already processed): %s" % response.request)
+
     def process_spider_output(self, response, result, spider):
         domain = spider.domain_name
-        self.filter.add(domain, response.request)
-
         for req in result:
             if isinstance(req, Request):
                 added = self.filter.add(domain, req)
                 if not (added or req.dont_filter):
-                    log.msg('Skipped (already visited): %s' % req, log.TRACE, domain=domain)
+                    log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain)
                     continue
             yield req
 

scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py

@@ -2,6 +2,7 @@ import unittest
 
 from scrapy.spider import spiders
 from scrapy.http import Request, Response
+from scrapy.core.exceptions import IgnoreRequest
 from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware, SimplePerDomainFilter
 
 class DuplicatesFilterMiddlewareTest(unittest.TestCase):
@@ -23,6 +24,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
 
+        mw.process_spider_input(response, self.spider)
         filtered = list(mw.process_spider_output(response, [r0, r1, r2, r3], self.spider))
 
         assert r0 not in filtered
@@ -30,6 +32,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
         assert r2 in filtered
         assert r3 not in filtered
 
+        response = Response('http://scrapytest.org/')
+        response.request = Request('http://scrapytest.org/')
+        self.assertRaises(IgnoreRequest, mw.process_spider_input, response, self.spider)
+
         mw.filter.close('scrapytest.org')