From d1265a4199f4ff22c2dd1805a7ab30bef03fba7f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 23 Feb 2009 20:21:11 +0000 Subject: [PATCH] move dupe filter code outside of core. refs #49 --HG-- rename : scrapy/trunk/scrapy/core/filters.py => scrapy/trunk/scrapy/dupefilter/__init__.py extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40901 --- scrapy/trunk/scrapy/conf/default_settings.py | 2 +- .../scrapy/contrib/downloadermiddleware/redirect.py | 8 -------- .../contrib/schedulermiddleware/duplicatesfilter.py | 7 ++----- .../contrib/spidermiddleware/duplicatesfilter.py | 5 ++--- .../scrapy/{core/filters.py => dupefilter/__init__.py} | 9 +++------ .../scrapy/tests/test_downloadermiddleware_redirect.py | 7 +++++-- scrapy/trunk/scrapy/tests/test_filters.py | 2 +- .../tests/test_schedulermiddleware_duplicatesfilter.py | 6 +++--- .../tests/test_spidermiddleware_duplicatesfilter.py | 10 +++++----- 9 files changed, 22 insertions(+), 34 deletions(-) rename scrapy/trunk/scrapy/{core/filters.py => dupefilter/__init__.py} (91%) diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index 6febc6d4b..85a6c5ed5 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -76,7 +76,7 @@ DOWNLOADER_MIDDLEWARES = [ DOWNLOADER_STATS = True -DUPLICATESFILTER_FILTERCLASS = 'scrapy.core.filters.SimplePerDomainFilter' +DUPEFILTER_FILTERCLASS = 'scrapy.dupefilter.SimplePerDomainFilter' ENABLED_SPIDERS_FILE = '' diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index 8de57a1db..e828d2e2e 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,15 +1,7 @@ -import re -from pydispatch import dispatcher - -from scrapy.core import signals from scrapy import log -from scrapy.http import Request, Response from scrapy.core.exceptions import HttpException -from scrapy.core.filters import duplicatesfilter -from scrapy.utils.misc import load_object from scrapy.utils.url import urljoin_rfc as urljoin from scrapy.utils.response import get_meta_refresh -from scrapy.conf import settings class RedirectLoop(Exception): pass diff --git a/scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py index 8b0a47c62..f6954058c 100644 --- a/scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py @@ -2,17 +2,14 @@ DuplicatesFilterMiddleware: Filter out already visited urls """ -from scrapy.http import Request -from scrapy.conf import settings from scrapy.core.exceptions import IgnoreRequest -from scrapy.core.filters import duplicatesfilter -from scrapy import log +from scrapy.dupefilter import dupefilter class DuplicatesFilterMiddleware(object): """Filter out already seen requests to avoid visiting pages more than once.""" def enqueue_request(self, domain, request, priority): - added = duplicatesfilter.add(domain, request) + added = dupefilter.add(domain, request) if not (added or request.dont_filter): raise IgnoreRequest('Skipped (already seen request') diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py index f327190a9..51e99d211 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py @@ -3,8 +3,7 @@ DuplicatesFilterMiddleware: Filter out already visited urls """ from scrapy.http import Request -from scrapy.conf import settings -from scrapy.core.filters import duplicatesfilter +from scrapy.dupefilter import dupefilter from scrapy import log @@ -15,7 +14,7 @@ class DuplicatesFilterMiddleware(object): domain = spider.domain_name for req in result: if isinstance(req, Request): - has = duplicatesfilter.has(domain, req) + has = dupefilter.has(domain, req) if has and not req.dont_filter: log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain) continue diff --git a/scrapy/trunk/scrapy/core/filters.py b/scrapy/trunk/scrapy/dupefilter/__init__.py similarity index 91% rename from scrapy/trunk/scrapy/core/filters.py rename to scrapy/trunk/scrapy/dupefilter/__init__.py index 9dc212ab8..54383dfff 100644 --- a/scrapy/trunk/scrapy/core/filters.py +++ b/scrapy/trunk/scrapy/dupefilter/__init__.py @@ -1,12 +1,9 @@ from pydispatch import dispatcher from scrapy.core import signals -from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import load_object from scrapy.conf import settings -from scrapy import log class BaseFilter(dict): @@ -78,9 +75,9 @@ class NullFilter(dict): try: - duplicatesfilter + dupefilter except NameError: - clspath = settings.get('DUPLICATESFILTER_FILTERCLASS') + clspath = settings.get('DUPEFILTER_FILTERCLASS') cls = load_object(clspath) if clspath else NullFilter - duplicatesfilter = cls() + dupefilter = cls() diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py index 42b42f38e..b696574aa 100644 --- a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py @@ -2,7 +2,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.core.exceptions import HttpException -from scrapy.core.filters import duplicatesfilter +from scrapy.dupefilter import dupefilter from scrapy.spider import spiders from scrapy.http import Request, Response, Headers @@ -12,7 +12,10 @@ class RedirectMiddlewareTest(unittest.TestCase): spiders.spider_modules = ['scrapy.tests.test_spiders'] spiders.reload() self.spider = spiders.fromdomain('scrapytest.org') - duplicatesfilter.open('scrapytest.org') + dupefilter.open('scrapytest.org') + + def tearDown(self): + dupefilter.close('scrapytest.org') def test_process_exception(self): diff --git a/scrapy/trunk/scrapy/tests/test_filters.py b/scrapy/trunk/scrapy/tests/test_filters.py index ba71b9614..bb3f7ec5a 100644 --- a/scrapy/trunk/scrapy/tests/test_filters.py +++ b/scrapy/trunk/scrapy/tests/test_filters.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.core.filters import SimplePerDomainFilter, NullFilter +from scrapy.dupefilter import SimplePerDomainFilter, NullFilter class SimplePerDomainFilterTest(unittest.TestCase): diff --git a/scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index dc52495d4..75becd859 100644 --- a/scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -4,16 +4,16 @@ from scrapy.spider import spiders from scrapy.http import Request, Response from scrapy.core.exceptions import IgnoreRequest from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware -from scrapy.core.filters import duplicatesfilter +from scrapy.dupefilter import dupefilter class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): - duplicatesfilter.open('scrapytest.org') + dupefilter.open('scrapytest.org') def tearDown(self): - duplicatesfilter.close('scrapytest.org') + dupefilter.close('scrapytest.org') def test_process_spider_output(self): domain = 'scrapytest.org' diff --git a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py index a9e405440..3ae448068 100644 --- a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py +++ b/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py @@ -4,7 +4,7 @@ from scrapy.spider import spiders from scrapy.http import Request, Response from scrapy.core.exceptions import IgnoreRequest from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware -from scrapy.core.filters import duplicatesfilter +from scrapy.dupefilter import dupefilter class DuplicatesFilterMiddlewareTest(unittest.TestCase): @@ -12,10 +12,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): spiders.spider_modules = ['scrapy.tests.test_spiders'] spiders.reload() self.spider = spiders.fromdomain('scrapytest.org') - duplicatesfilter.open('scrapytest.org') + dupefilter.open('scrapytest.org') def tearDown(self): - duplicatesfilter.close('scrapytest.org') + dupefilter.close('scrapytest.org') def test_process_spider_output(self): mw = DuplicatesFilterMiddleware() @@ -28,8 +28,8 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') - duplicatesfilter.add('scrapytest.org', r0) - duplicatesfilter.add('scrapytest.org', r2) + dupefilter.add('scrapytest.org', r0) + dupefilter.add('scrapytest.org', r2) filtered = list(mw.process_spider_output(response, [r0, r1, r2, r3], self.spider))