Sorted out Duplicate Filter API.

--HG--
rename : scrapy/dupefilter/__init__.py => scrapy/contrib/dupefilter.py

docs/ref/settings.rst

@@ -383,17 +383,17 @@ Default: ``180``
 
 The amount of time (in secs) that the downloader will wait before timing out.
 
-.. setting:: DUPEFILTER_FILTERCLASS
+.. setting:: DUPEFILTER_CLASS
 
-DUPEFILTER_FILTERCLASS
-----------------------------
+DUPEFILTER_CLASS
+----------------
 
-Default: ``scrapy.contrib.spidermiddleware.SimplePerDomainFilter``
+Default: ``'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'``
 
-The class used to detect and filter duplicated requests.
+The class used to detect and filter duplicate requests.
 
-Default ``SimplePerDomainFilter`` filter based on request fingerprint and
-grouping per domain.
+The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint
+(using ``scrapy.utils.request.request_fingerprint``) and grouping per domain.
 
 .. setting:: ENGINE_DEBUG
 

scrapy/conf/default_settings.py

@@ -80,7 +80,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
 
 DOWNLOADER_STATS = True
 
-DUPEFILTER_FILTERCLASS = 'scrapy.dupefilter.SimplePerDomainFilter'
+DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'
 
 ENABLED_SPIDERS_FILE = ''
 

scrapy/contrib/dupefilter.py

@@ -0,0 +1,49 @@
+"""
+Dupe Filter classes implement a mechanism for filtering duplicate requests.
+They must implement the following methods:
+
+* open_domain(domain)
+  open a domain for tracking duplicates (typically used to reserve resources)
+
+* close_domain(domain)
+  close a domain (typically used for freeing resources)
+
+* request_seen(domain, request, dont_record=False)
+  return ``True`` if the request was seen before, or ``False`` otherwise. If
+  ``dont_record`` is ``True`` the request must not be recorded as seen.
+
+"""
+
+from scrapy.utils.request import request_fingerprint
+
+
+class NullDupeFilter(dict):
+    def open_domain(self, domain):
+        pass
+
+    def close_domain(self, domain):
+        pass
+
+    def request_seen(self, domain, request, dont_record=False):
+        return False
+
+
+class RequestFingerprintDupeFilter(object):
+    """Duplicate filter using scrapy.utils.request.request_fingerprint"""
+
+    def __init__(self):
+        self.fingerprints = {}
+
+    def open_domain(self, domain):
+        self.fingerprints[domain] = set()
+
+    def close_domain(self, domain):
+        del self.fingerprints[domain]
+
+    def request_seen(self, domain, request, dont_record=False):
+        fp = request_fingerprint(request)
+        if fp in self.fingerprints[domain]:
+            return True
+        if not dont_record:
+            self.fingerprints[domain].add(fp)
+        return False

scrapy/contrib/schedulermiddleware/duplicatesfilter.py

@@ -9,19 +9,19 @@ from scrapy.conf import settings
 class DuplicatesFilterMiddleware(object):
     """Filter out already seen requests to avoid visiting pages more than once."""
     def __init__(self):
-        clspath = settings.get('DUPEFILTER_FILTERCLASS')
+        clspath = settings.get('DUPEFILTER_CLASS')
         if not clspath:
             raise NotConfigured
 
         self.dupefilter = load_object(clspath)()
 
     def enqueue_request(self, domain, request):
-        added = self.dupefilter.add(domain, request)
-        if not (added or request.dont_filter):
-            raise IgnoreRequest('Skipped (already seen request)')
+        seen = self.dupefilter.request_seen(domain, request)
+        if seen and not request.dont_filter:
+            raise IgnoreRequest('Skipped (request already seen)')
 
     def open_domain(self, domain):
-        self.dupefilter.open(domain)
+        self.dupefilter.open_domain(domain)
 
     def close_domain(self, domain):
-        self.dupefilter.close(domain)
+        self.dupefilter.close_domain(domain)

scrapy/dupefilter/__init__.py

@@ -1,66 +0,0 @@
-from scrapy.utils.request import request_fingerprint
-
-
-class BaseFilter(dict):
-    """Base class defining the duplicates requests filtering api"""
-
-    def open(self, domain):
-        """Called when a domain starts"""
-        raise NotImplementedError()
-
-    def close(self, domain):
-        """Called when a domain is closed"""
-        raise NotImplementedError()
-
-    def add(self, domain, request):
-        """Called to check if a request was already seen, and adds it to seen set.
-
-        returns True if not seen before, or False otherwise.
-        """
-        raise NotImplementedError()
-
-    def has(self, domain, request):
-        """Called to check if a request was seen but doesnt add request to seen set."""
-        raise NotImplementedError()
-
-
-class SimplePerDomainFilter(BaseFilter):
-    """Filter out a request if already seen for same domain"""
-
-    def open(self, domain):
-        """Initialize the resources needed for filtering for this domain"""
-        self[domain] = set()
-
-    def close(self, domain):
-        """Remove the resources reserved for filtering for this domain"""
-        del self[domain]
-
-    def add(self, domain, request):
-        """Add the fingerprint of a request to the domain set if a equivalent fingerprint has not been added.
-        This method will return true if the fingerprint was added and false otherwise.
-        """
-        fp = request_fingerprint(request)
-        if fp not in self[domain]:
-            self[domain].add(fp)
-            return True
-        return False
-
-    def has(self, domain, request):
-        fp = request_fingerprint(request)
-        return fp in self[domain]
-
-
-class NullFilter(dict):
-    def open(self, domain):
-        pass
-
-    def close(self, domain):
-        pass
-
-    def add(self, domain, request):
-        return True
-
-    def has(self, domain, request):
-        return None
-
-

scrapy/tests/test_dupefilter.py

@@ -1,41 +1,36 @@
 import unittest
 
 from scrapy.http import Request
-from scrapy.dupefilter import SimplePerDomainFilter, NullFilter
+from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter
 
 
-class SimplePerDomainFilterTest(unittest.TestCase):
+class RequestFingerprintDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
         domain = 'scrapytest.org'
-        filter = SimplePerDomainFilter()
-        filter.open(domain)
-        assert domain in filter
+        filter = RequestFingerprintDupeFilter()
+        filter.open_domain(domain)
 
         r1 = Request('http://scrapytest.org/1')
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
 
-        assert not filter.has(domain, r1)
-        assert filter.add(domain, r1)
-        assert filter.has(domain, r1)
+        assert not filter.request_seen(domain, r1)
+        assert filter.request_seen(domain, r1)
 
-        assert filter.add(domain, r2)
-        assert not filter.add(domain, r3)
+        assert not filter.request_seen(domain, r2)
+        assert filter.request_seen(domain, r3)
 
-        filter.close(domain)
-        assert domain not in filter
+        filter.close_domain(domain)
 
 
-class NullFilterTest(unittest.TestCase):
+class NullDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
         domain = 'scrapytest.org'
-        filter = NullFilter()
-        filter.open(domain)
+        filter = NullDupeFilter()
+        filter.open_domain(domain)
 
         r1 = Request('http://scrapytest.org/1')
-        assert not filter.has(domain, r1)
-        assert filter.add(domain, r1)
-        assert not filter.has(domain, r1)
-        filter.close(domain)
+        assert not filter.request_seen(domain, r1)
+        filter.close_domain(domain)