diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py index 369cbf89c..1760006cf 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py @@ -39,8 +39,7 @@ class DuplicatesFilterMiddleware(object): dispatcher.connect(self.filter.close, signals.domain_closed) def process_spider_input(self, response, spider): - if not self.filter.add(spider.domain_name, response.request): - raise IgnoreRequest("Skipped (already processed): %s" % response.request) + self.filter.add(spider.domain_name, response.request) def process_spider_output(self, response, result, spider): domain = spider.domain_name diff --git a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py index 11325437b..606ff1687 100644 --- a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py +++ b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py @@ -32,10 +32,6 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): assert r2 in filtered assert r3 not in filtered - response = Response('http://scrapytest.org/') - response.request = Request('http://scrapytest.org/') - self.assertRaises(IgnoreRequest, mw.process_spider_input, response, self.spider) - mw.filter.close('scrapytest.org')