From fe7935005c5b6cad1983d0e82a4610225cbb5a23 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 19 Feb 2009 16:37:33 +0000 Subject: [PATCH] duplicatefilter: filter request prior to reach spider --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40884 --- .../contrib/spidermiddleware/duplicatesfilter.py | 10 ++++++---- .../tests/test_spidermiddleware_duplicatesfilter.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py index e6a8872ab..369cbf89c 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py @@ -6,7 +6,7 @@ from pydispatch import dispatcher from scrapy.core import signals from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured +from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import load_object from scrapy.conf import settings @@ -38,15 +38,17 @@ class DuplicatesFilterMiddleware(object): dispatcher.connect(self.filter.open, signals.domain_open) dispatcher.connect(self.filter.close, signals.domain_closed) + def process_spider_input(self, response, spider): + if not self.filter.add(spider.domain_name, response.request): + raise IgnoreRequest("Skipped (already processed): %s" % response.request) + def process_spider_output(self, response, result, spider): domain = spider.domain_name - self.filter.add(domain, response.request) - for req in result: if isinstance(req, Request): added = self.filter.add(domain, req) if not (added or req.dont_filter): - log.msg('Skipped (already visited): %s' % req, log.TRACE, domain=domain) + log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain) continue yield req diff --git a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py index d3e7fb11f..11325437b 100644 --- a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py +++ b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py @@ -2,6 +2,7 @@ import unittest from scrapy.spider import spiders from scrapy.http import Request, Response +from scrapy.core.exceptions import IgnoreRequest from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware, SimplePerDomainFilter class DuplicatesFilterMiddlewareTest(unittest.TestCase): @@ -23,6 +24,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') + mw.process_spider_input(response, self.spider) filtered = list(mw.process_spider_output(response, [r0, r1, r2, r3], self.spider)) assert r0 not in filtered @@ -30,6 +32,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): assert r2 in filtered assert r3 not in filtered + response = Response('http://scrapytest.org/') + response.request = Request('http://scrapytest.org/') + self.assertRaises(IgnoreRequest, mw.process_spider_input, response, self.spider) + mw.filter.close('scrapytest.org')