From adccd9a04ea3e1bfccbd42bce5b92002ecb95551 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 20 Jun 2009 20:29:07 -0300 Subject: [PATCH] Sorted out Duplicate Filter API. --HG-- rename : scrapy/dupefilter/__init__.py => scrapy/contrib/dupefilter.py --- docs/ref/settings.rst | 14 ++-- scrapy/conf/default_settings.py | 2 +- scrapy/contrib/dupefilter.py | 49 ++++++++++++++ .../schedulermiddleware/duplicatesfilter.py | 12 ++-- scrapy/dupefilter/__init__.py | 66 ------------------- scrapy/tests/test_dupefilter.py | 33 ++++------ 6 files changed, 77 insertions(+), 99 deletions(-) create mode 100644 scrapy/contrib/dupefilter.py delete mode 100644 scrapy/dupefilter/__init__.py diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 60bea9154..80166f541 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -383,17 +383,17 @@ Default: ``180`` The amount of time (in secs) that the downloader will wait before timing out. -.. setting:: DUPEFILTER_FILTERCLASS +.. setting:: DUPEFILTER_CLASS -DUPEFILTER_FILTERCLASS ----------------------------- +DUPEFILTER_CLASS +---------------- -Default: ``scrapy.contrib.spidermiddleware.SimplePerDomainFilter`` +Default: ``'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'`` -The class used to detect and filter duplicated requests. +The class used to detect and filter duplicate requests. -Default ``SimplePerDomainFilter`` filter based on request fingerprint and -grouping per domain. +The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint +(using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. .. setting:: ENGINE_DEBUG diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 9a1420f10..f8de29128 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -80,7 +80,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { DOWNLOADER_STATS = True -DUPEFILTER_FILTERCLASS = 'scrapy.dupefilter.SimplePerDomainFilter' +DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' ENABLED_SPIDERS_FILE = '' diff --git a/scrapy/contrib/dupefilter.py b/scrapy/contrib/dupefilter.py new file mode 100644 index 000000000..1e8c336fc --- /dev/null +++ b/scrapy/contrib/dupefilter.py @@ -0,0 +1,49 @@ +""" +Dupe Filter classes implement a mechanism for filtering duplicate requests. +They must implement the following methods: + +* open_domain(domain) + open a domain for tracking duplicates (typically used to reserve resources) + +* close_domain(domain) + close a domain (typically used for freeing resources) + +* request_seen(domain, request, dont_record=False) + return ``True`` if the request was seen before, or ``False`` otherwise. If + ``dont_record`` is ``True`` the request must not be recorded as seen. + +""" + +from scrapy.utils.request import request_fingerprint + + +class NullDupeFilter(dict): + def open_domain(self, domain): + pass + + def close_domain(self, domain): + pass + + def request_seen(self, domain, request, dont_record=False): + return False + + +class RequestFingerprintDupeFilter(object): + """Duplicate filter using scrapy.utils.request.request_fingerprint""" + + def __init__(self): + self.fingerprints = {} + + def open_domain(self, domain): + self.fingerprints[domain] = set() + + def close_domain(self, domain): + del self.fingerprints[domain] + + def request_seen(self, domain, request, dont_record=False): + fp = request_fingerprint(request) + if fp in self.fingerprints[domain]: + return True + if not dont_record: + self.fingerprints[domain].add(fp) + return False diff --git a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py index dff52cae4..c35037d4f 100644 --- a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py +++ b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py @@ -9,19 +9,19 @@ from scrapy.conf import settings class DuplicatesFilterMiddleware(object): """Filter out already seen requests to avoid visiting pages more than once.""" def __init__(self): - clspath = settings.get('DUPEFILTER_FILTERCLASS') + clspath = settings.get('DUPEFILTER_CLASS') if not clspath: raise NotConfigured self.dupefilter = load_object(clspath)() def enqueue_request(self, domain, request): - added = self.dupefilter.add(domain, request) - if not (added or request.dont_filter): - raise IgnoreRequest('Skipped (already seen request)') + seen = self.dupefilter.request_seen(domain, request) + if seen and not request.dont_filter: + raise IgnoreRequest('Skipped (request already seen)') def open_domain(self, domain): - self.dupefilter.open(domain) + self.dupefilter.open_domain(domain) def close_domain(self, domain): - self.dupefilter.close(domain) + self.dupefilter.close_domain(domain) diff --git a/scrapy/dupefilter/__init__.py b/scrapy/dupefilter/__init__.py deleted file mode 100644 index c9763e645..000000000 --- a/scrapy/dupefilter/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -from scrapy.utils.request import request_fingerprint - - -class BaseFilter(dict): - """Base class defining the duplicates requests filtering api""" - - def open(self, domain): - """Called when a domain starts""" - raise NotImplementedError() - - def close(self, domain): - """Called when a domain is closed""" - raise NotImplementedError() - - def add(self, domain, request): - """Called to check if a request was already seen, and adds it to seen set. - - returns True if not seen before, or False otherwise. - """ - raise NotImplementedError() - - def has(self, domain, request): - """Called to check if a request was seen but doesnt add request to seen set.""" - raise NotImplementedError() - - -class SimplePerDomainFilter(BaseFilter): - """Filter out a request if already seen for same domain""" - - def open(self, domain): - """Initialize the resources needed for filtering for this domain""" - self[domain] = set() - - def close(self, domain): - """Remove the resources reserved for filtering for this domain""" - del self[domain] - - def add(self, domain, request): - """Add the fingerprint of a request to the domain set if a equivalent fingerprint has not been added. - This method will return true if the fingerprint was added and false otherwise. - """ - fp = request_fingerprint(request) - if fp not in self[domain]: - self[domain].add(fp) - return True - return False - - def has(self, domain, request): - fp = request_fingerprint(request) - return fp in self[domain] - - -class NullFilter(dict): - def open(self, domain): - pass - - def close(self, domain): - pass - - def add(self, domain, request): - return True - - def has(self, domain, request): - return None - - diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index bb3f7ec5a..99e67dc21 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -1,41 +1,36 @@ import unittest from scrapy.http import Request -from scrapy.dupefilter import SimplePerDomainFilter, NullFilter +from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter -class SimplePerDomainFilterTest(unittest.TestCase): +class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): domain = 'scrapytest.org' - filter = SimplePerDomainFilter() - filter.open(domain) - assert domain in filter + filter = RequestFingerprintDupeFilter() + filter.open_domain(domain) r1 = Request('http://scrapytest.org/1') r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') - assert not filter.has(domain, r1) - assert filter.add(domain, r1) - assert filter.has(domain, r1) + assert not filter.request_seen(domain, r1) + assert filter.request_seen(domain, r1) - assert filter.add(domain, r2) - assert not filter.add(domain, r3) + assert not filter.request_seen(domain, r2) + assert filter.request_seen(domain, r3) - filter.close(domain) - assert domain not in filter + filter.close_domain(domain) -class NullFilterTest(unittest.TestCase): +class NullDupeFilterTest(unittest.TestCase): def test_filter(self): domain = 'scrapytest.org' - filter = NullFilter() - filter.open(domain) + filter = NullDupeFilter() + filter.open_domain(domain) r1 = Request('http://scrapytest.org/1') - assert not filter.has(domain, r1) - assert filter.add(domain, r1) - assert not filter.has(domain, r1) - filter.close(domain) + assert not filter.request_seen(domain, r1) + filter.close_domain(domain)