move dupe filter code outside of core. refs #49

--HG--
rename : scrapy/trunk/scrapy/core/filters.py => scrapy/trunk/scrapy/dupefilter/__init__.py
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40901

scrapy/trunk/scrapy/conf/default_settings.py

@@ -76,7 +76,7 @@ DOWNLOADER_MIDDLEWARES = [
 
 DOWNLOADER_STATS = True
 
-DUPLICATESFILTER_FILTERCLASS = 'scrapy.core.filters.SimplePerDomainFilter'
+DUPEFILTER_FILTERCLASS = 'scrapy.dupefilter.SimplePerDomainFilter'
 
 ENABLED_SPIDERS_FILE = ''
 

scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py

@@ -1,15 +1,7 @@
-import re
-from pydispatch import dispatcher
-
-from scrapy.core import signals
 from scrapy import log
-from scrapy.http import Request, Response
 from scrapy.core.exceptions import HttpException
-from scrapy.core.filters import duplicatesfilter
-from scrapy.utils.misc import load_object
 from scrapy.utils.url import urljoin_rfc as urljoin
 from scrapy.utils.response import get_meta_refresh
-from scrapy.conf import settings
 
 class RedirectLoop(Exception):
     pass

scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py

@@ -2,17 +2,14 @@
 DuplicatesFilterMiddleware: Filter out already visited urls
 """
 
-from scrapy.http import Request
-from scrapy.conf import settings
 from scrapy.core.exceptions import IgnoreRequest
-from scrapy.core.filters import duplicatesfilter
-from scrapy import log
+from scrapy.dupefilter import dupefilter
 
 
 class DuplicatesFilterMiddleware(object):
     """Filter out already seen requests to avoid visiting pages more than once."""
 
     def enqueue_request(self, domain, request, priority):
-        added = duplicatesfilter.add(domain, request)
+        added = dupefilter.add(domain, request)
         if not (added or request.dont_filter):
             raise IgnoreRequest('Skipped (already seen request')

scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py

@@ -3,8 +3,7 @@ DuplicatesFilterMiddleware: Filter out already visited urls
 """
 
 from scrapy.http import Request
-from scrapy.conf import settings
-from scrapy.core.filters import duplicatesfilter
+from scrapy.dupefilter import dupefilter
 from scrapy import log
 
 
@@ -15,7 +14,7 @@ class DuplicatesFilterMiddleware(object):
         domain = spider.domain_name
         for req in result:
             if isinstance(req, Request):
-                has = duplicatesfilter.has(domain, req)
+                has = dupefilter.has(domain, req)
                 if has and not req.dont_filter:
                     log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain)
                     continue

scrapy/trunk/scrapy/dupefilter/__init__.py

@@ -1,12 +1,9 @@
 from pydispatch import dispatcher
 
 from scrapy.core import signals
-from scrapy.http import Request
-from scrapy.core.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import load_object
 from scrapy.conf import settings
-from scrapy import log
 
 
 class BaseFilter(dict):
@@ -78,9 +75,9 @@ class NullFilter(dict):
 
 
 try:
-    duplicatesfilter
+    dupefilter
 except NameError:
-    clspath = settings.get('DUPLICATESFILTER_FILTERCLASS')
+    clspath = settings.get('DUPEFILTER_FILTERCLASS')
     cls = load_object(clspath) if clspath else NullFilter
-    duplicatesfilter = cls()
+    dupefilter = cls()
 

scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py

@@ -2,7 +2,7 @@ import unittest
 
 from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
 from scrapy.core.exceptions import HttpException
-from scrapy.core.filters import duplicatesfilter
+from scrapy.dupefilter import dupefilter
 from scrapy.spider import spiders
 from scrapy.http import Request, Response, Headers
 
@@ -12,7 +12,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
         spiders.spider_modules = ['scrapy.tests.test_spiders']
         spiders.reload()
         self.spider = spiders.fromdomain('scrapytest.org')
-        duplicatesfilter.open('scrapytest.org')
+        dupefilter.open('scrapytest.org')
+
+    def tearDown(self):
+        dupefilter.close('scrapytest.org')
 
     def test_process_exception(self):
 

scrapy/trunk/scrapy/tests/test_filters.py

@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.http import Request
-from scrapy.core.filters import SimplePerDomainFilter, NullFilter
+from scrapy.dupefilter import SimplePerDomainFilter, NullFilter
 
 
 class SimplePerDomainFilterTest(unittest.TestCase):

scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py

@@ -4,16 +4,16 @@ from scrapy.spider import spiders
 from scrapy.http import Request, Response
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
-from scrapy.core.filters import duplicatesfilter
+from scrapy.dupefilter import dupefilter
 
 
 class DuplicatesFilterMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        duplicatesfilter.open('scrapytest.org')
+        dupefilter.open('scrapytest.org')
 
     def tearDown(self):
-        duplicatesfilter.close('scrapytest.org')
+        dupefilter.close('scrapytest.org')
 
     def test_process_spider_output(self):
         domain = 'scrapytest.org'

scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py

@@ -4,7 +4,7 @@ from scrapy.spider import spiders
 from scrapy.http import Request, Response
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
-from scrapy.core.filters import duplicatesfilter
+from scrapy.dupefilter import dupefilter
 
 class DuplicatesFilterMiddlewareTest(unittest.TestCase):
 
@@ -12,10 +12,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
         spiders.spider_modules = ['scrapy.tests.test_spiders']
         spiders.reload()
         self.spider = spiders.fromdomain('scrapytest.org')
-        duplicatesfilter.open('scrapytest.org')
+        dupefilter.open('scrapytest.org')
 
     def tearDown(self):
-        duplicatesfilter.close('scrapytest.org')
+        dupefilter.close('scrapytest.org')
 
     def test_process_spider_output(self):
         mw = DuplicatesFilterMiddleware()
@@ -28,8 +28,8 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
 
-        duplicatesfilter.add('scrapytest.org', r0)
-        duplicatesfilter.add('scrapytest.org', r2)
+        dupefilter.add('scrapytest.org', r0)
+        dupefilter.add('scrapytest.org', r2)
 
         filtered = list(mw.process_spider_output(response, [r0, r1, r2, r3], self.spider))