From 0e98526daa08dbdfbd8dc42d8447935f437b84f5 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 19 Feb 2009 19:43:24 +0000 Subject: [PATCH] duplicatesfilter: add a singeton duplicates filter and adapt current middleware and redirection middleware --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40889 --- scrapy/trunk/scrapy/conf/default_settings.py | 2 +- .../contrib/downloadermiddleware/redirect.py | 22 ++++++-- .../spidermiddleware/duplicatesfilter.py | 55 ++----------------- .../test_downloadermiddleware_redirect.py | 6 +- .../test_spidermiddleware_duplicatesfilter.py | 28 +--------- 5 files changed, 28 insertions(+), 85 deletions(-) diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index 169cd3502..4bf2aa07f 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -76,7 +76,7 @@ DOWNLOADER_MIDDLEWARES = [ DOWNLOADER_STATS = True -DUPLICATESFILTER_FILTERCLASS = 'scrapy.contrib.spidermiddleware.duplicatesfilter.SimplePerDomainFilter' +DUPLICATESFILTER_FILTERCLASS = 'scrapy.core.filters.SimplePerDomainFilter' ENABLED_SPIDERS_FILE = '' diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index c81fbc8c8..d2e8f5a54 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,10 +1,15 @@ import re +from pydispatch import dispatcher +from scrapy.core import signals from scrapy import log from scrapy.http import Request, Response from scrapy.core.exceptions import HttpException +from scrapy.core.filters import duplicatesfilter +from scrapy.utils.misc import load_object from scrapy.utils.url import urljoin_rfc as urljoin from scrapy.utils.response import get_meta_refresh +from scrapy.conf import settings class RedirectLoop(Exception): pass @@ -19,26 +24,31 @@ class RedirectMiddleware(object): if not isinstance(exception, HttpException): return + domain = spider.domain_name status = exception.status response = exception.response if status in set([302, 303]): redirected_url = urljoin(request.url, response.headers['location'][0]) redirected = request.replace(url=redirected_url, method='GET', body=None) - log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected + return self._redirect(redirected, request, spider, status) if status in [301, 307]: redirected_url = urljoin(request.url, response.headers['location'][0]) redirected = request.replace(url=redirected_url) - log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected + return self._redirect(redirected, request, spider, status) def process_response(self, request, response, spider): interval, url = get_meta_refresh(response) if url and int(interval) < META_REFRESH_MAXSEC: redirected = request.replace(url=urljoin(request.url, url)) - log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected + return self._redirect(redirected, request, spider, 'meta refresh') or response return response + + def _redirect(self, redirected, request, spider, reason): + domain = spider.domain_name + if duplicatesfilter.add(domain, redirected) or redirected.dont_filter: + log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request), level=log.DEBUG, domain=domain) + return redirected + diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py index 1760006cf..eec2dbd86 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py @@ -2,73 +2,26 @@ DuplicatesFilterMiddleware: Filter out already visited urls """ -from pydispatch import dispatcher - -from scrapy.core import signals from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured, IgnoreRequest -from scrapy.utils.request import request_fingerprint -from scrapy.utils.misc import load_object from scrapy.conf import settings +from scrapy.core.filters import duplicatesfilter from scrapy import log class DuplicatesFilterMiddleware(object): - """Filter out duplicate requests to avoid visiting same page more than once. - - filter class (defined by DUPLICATESFILTER_FILTERCLASS setting) must - inplement a simple API: - - * open(domain) called when a new domain starts - - * close(domain) called when a domain is going to be closed. - - * add(domain, request) called each time a new request needs to be tested - looking if it is was already seen or is a new one. This is the most - important method for a filtering class. - - """ - - def __init__(self): - clspath = settings.get('DUPLICATESFILTER_FILTERCLASS') - if not clspath: - raise NotConfigured - - self.filter = load_object(clspath)() - dispatcher.connect(self.filter.open, signals.domain_open) - dispatcher.connect(self.filter.close, signals.domain_closed) + """Filter out already seen requests to avoid visiting pages more than once.""" def process_spider_input(self, response, spider): - self.filter.add(spider.domain_name, response.request) + duplicatesfilter.add(spider.domain_name, response.request) def process_spider_output(self, response, result, spider): domain = spider.domain_name for req in result: if isinstance(req, Request): - added = self.filter.add(domain, req) + added = duplicatesfilter.add(domain, req) if not (added or req.dont_filter): log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain) continue yield req -class SimplePerDomainFilter(dict): - """Filter out a request if already seen for same domain""" - - def open(self, domain): - """Initialize the resources needed for filtering for this domain""" - self[domain] = set() - - def close(self, domain): - """Remove the resources reserved for filtering for this domain""" - del self[domain] - - def add(self, domain, request): - """Add the fingerprint of a request to the domain set if a equivalent fingerprint has not been added. - This method will return true if the fingerprint was added and false otherwise. - """ - fp = request_fingerprint(request) - if fp not in self[domain]: - self[domain].add(fp) - return True - return False diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py index 7fd0e98ff..42b42f38e 100644 --- a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py @@ -2,6 +2,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.core.exceptions import HttpException +from scrapy.core.filters import duplicatesfilter from scrapy.spider import spiders from scrapy.http import Request, Response, Headers @@ -11,6 +12,7 @@ class RedirectMiddlewareTest(unittest.TestCase): spiders.spider_modules = ['scrapy.tests.test_spiders'] spiders.reload() self.spider = spiders.fromdomain('scrapytest.org') + duplicatesfilter.open('scrapytest.org') def test_process_exception(self): @@ -28,7 +30,7 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url2) url = 'http://www.example.com/302' - url2 = 'http://www.example.com/redirected' + url2 = 'http://www.example.com/redirected2' req = Request(url, method='POST') hdr = Headers({'Location': [url2]}) rsp = Response(url, headers=hdr) @@ -43,7 +45,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_process_response(self): mw = RedirectMiddleware() - + body = """ """ diff --git a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py index 606ff1687..2d6ef03b0 100644 --- a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py +++ b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py @@ -3,7 +3,8 @@ import unittest from scrapy.spider import spiders from scrapy.http import Request, Response from scrapy.core.exceptions import IgnoreRequest -from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware, SimplePerDomainFilter +from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware +from scrapy.core.filters import duplicatesfilter class DuplicatesFilterMiddlewareTest(unittest.TestCase): @@ -11,10 +12,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): spiders.spider_modules = ['scrapy.tests.test_spiders'] spiders.reload() self.spider = spiders.fromdomain('scrapytest.org') + duplicatesfilter.open('scrapytest.org') def test_process_spider_output(self): mw = DuplicatesFilterMiddleware() - mw.filter.open('scrapytest.org') response = Response('http://scrapytest.org/') response.request = Request('http://scrapytest.org/') @@ -31,26 +32,3 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): assert r1 in filtered assert r2 in filtered assert r3 not in filtered - - mw.filter.close('scrapytest.org') - - -class SimplePerDomainFilterTest(unittest.TestCase): - - def test_filter(self): - domain = 'scrapytest.org' - filter = SimplePerDomainFilter() - filter.open(domain) - assert domain in filter - - r1 = Request('http://scrapytest.org/1') - r2 = Request('http://scrapytest.org/2') - r3 = Request('http://scrapytest.org/2') - - assert filter.add(domain, r1) - assert filter.add(domain, r2) - assert not filter.add(domain, r3) - - filter.close(domain) - assert domain not in filter -