From 99467d4e6e485fa24467b4b02230028d924c4055 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 13 Sep 2009 20:51:43 -0300 Subject: [PATCH] Changed (unstable) scheduler middleware API to receive spider (instead of domain) in enqueue_request method --- docs/experimental/scheduler-middleware.rst | 6 ++-- scrapy/contrib/dupefilter.py | 30 +++++++++---------- .../schedulermiddleware/duplicatesfilter.py | 12 ++++---- scrapy/core/engine.py | 1 + scrapy/core/scheduler/middleware.py | 12 ++++---- scrapy/tests/test_dupefilter.py | 23 +++++++------- ...st_schedulermiddleware_duplicatesfilter.py | 16 +++++----- 7 files changed, 51 insertions(+), 49 deletions(-) diff --git a/docs/experimental/scheduler-middleware.rst b/docs/experimental/scheduler-middleware.rst index 2956e84dd..dff8dede2 100644 --- a/docs/experimental/scheduler-middleware.rst +++ b/docs/experimental/scheduler-middleware.rst @@ -56,7 +56,7 @@ single Python class that defines one or more of the following methods: .. class:: SchedulerMiddleware - .. method:: enqueue_request(domain, request) + .. method:: enqueue_request(spider, request) Process the given request which is being enqueued. This method can return None to avoid the request from being scheduled. @@ -64,8 +64,8 @@ single Python class that defines one or more of the following methods: :meth:`enqueue_request` should return either ``None``, a :class:`~scrapy.http.Response` object or a ``Deferred``. - :param domain: the domain originating the request - :type domain: string + :param spider: the spider originating the request + :type spider: :class:`~scrapy.spider.BaseSpider` object :param requests: the request to be enqueued :type request: :class:`~scrapy.http.Request` object diff --git a/scrapy/contrib/dupefilter.py b/scrapy/contrib/dupefilter.py index 1e8c336fc..c6b9d185a 100644 --- a/scrapy/contrib/dupefilter.py +++ b/scrapy/contrib/dupefilter.py @@ -2,13 +2,13 @@ Dupe Filter classes implement a mechanism for filtering duplicate requests. They must implement the following methods: -* open_domain(domain) - open a domain for tracking duplicates (typically used to reserve resources) +* open_spider(spider) + open a spider for tracking duplicates (typically used to reserve resources) -* close_domain(domain) - close a domain (typically used for freeing resources) +* close_spider(spider) + close a spider (typically used for freeing resources) -* request_seen(domain, request, dont_record=False) +* request_seen(spider, request, dont_record=False) return ``True`` if the request was seen before, or ``False`` otherwise. If ``dont_record`` is ``True`` the request must not be recorded as seen. @@ -18,13 +18,13 @@ from scrapy.utils.request import request_fingerprint class NullDupeFilter(dict): - def open_domain(self, domain): + def open_spider(self, spider): pass - def close_domain(self, domain): + def close_spider(self, spider): pass - def request_seen(self, domain, request, dont_record=False): + def request_seen(self, spider, request, dont_record=False): return False @@ -34,16 +34,16 @@ class RequestFingerprintDupeFilter(object): def __init__(self): self.fingerprints = {} - def open_domain(self, domain): - self.fingerprints[domain] = set() + def open_spider(self, spider): + self.fingerprints[spider] = set() - def close_domain(self, domain): - del self.fingerprints[domain] + def close_spider(self, spider): + del self.fingerprints[spider] - def request_seen(self, domain, request, dont_record=False): + def request_seen(self, spider, request, dont_record=False): fp = request_fingerprint(request) - if fp in self.fingerprints[domain]: + if fp in self.fingerprints[spider]: return True if not dont_record: - self.fingerprints[domain].add(fp) + self.fingerprints[spider].add(fp) return False diff --git a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py index d9fff5c00..b610d41d6 100644 --- a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py +++ b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py @@ -17,13 +17,13 @@ class DuplicatesFilterMiddleware(object): self.dupefilter = load_object(clspath)() - def enqueue_request(self, domain, request): - seen = self.dupefilter.request_seen(domain, request) + def enqueue_request(self, spider, request): + seen = self.dupefilter.request_seen(spider, request) if seen and not request.dont_filter: raise IgnoreRequest('Skipped (request already seen)') - def open_domain(self, domain): - self.dupefilter.open_domain(domain) + def open_spider(self, spider): + self.dupefilter.open_spider(spider) - def close_domain(self, domain): - self.dupefilter.close_domain(domain) + def close_spider(self, spider): + self.dupefilter.close_spider(spider) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 44f982e02..62024d103 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -161,6 +161,7 @@ class ExecutionEngine(object): def crawl(self, request, spider): schd = mustbe_deferred(self.schedule, request, spider) + schd.addErrback(log.err, "Unhandled error on engine.crawl()") schd.addBoth(self.scraper.enqueue_scrape, request, spider) schd.addErrback(log.err, "Unhandled error on engine.crawl()") schd.addBoth(lambda _: self.next_request(spider)) diff --git a/scrapy/core/scheduler/middleware.py b/scrapy/core/scheduler/middleware.py index 7587f8504..2547aaff7 100644 --- a/scrapy/core/scheduler/middleware.py +++ b/scrapy/core/scheduler/middleware.py @@ -46,7 +46,7 @@ class SchedulerMiddlewareManager(object): self.loaded = True def _add_middleware(self, mw): - for name in ['enqueue_request', 'open_domain', 'close_domain']: + for name in ['enqueue_request', 'open_spider', 'close_spider']: mwfunc = getattr(mw, name, None) if mwfunc: self.mw_cbs[name].append(mwfunc) @@ -54,7 +54,7 @@ class SchedulerMiddlewareManager(object): def enqueue_request(self, wrappedfunc, spider, request): def _enqueue_request(request): for mwfunc in self.mw_cbs['enqueue_request']: - result = mwfunc(domain=spider.domain_name, request=request) + result = mwfunc(spider=spider, request=request) assert result is None or isinstance(result, (Response, Deferred)), \ 'Middleware %s.enqueue_request must return None, Response or Deferred, got %s' % \ (mwfunc.im_self.__class__.__name__, result.__class__.__name__) @@ -66,10 +66,10 @@ class SchedulerMiddlewareManager(object): return deferred def open_spider(self, spider): - for mwfunc in self.mw_cbs['open_domain']: - mwfunc(spider.domain_name) + for mwfunc in self.mw_cbs['open_spider']: + mwfunc(spider) def close_spider(self, spider): - for mwfunc in self.mw_cbs['close_domain']: - mwfunc(spider.domain_name) + for mwfunc in self.mw_cbs['close_spider']: + mwfunc(spider) diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index 99e67dc21..d11595a9d 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -1,36 +1,37 @@ import unittest from scrapy.http import Request +from scrapy.spider import BaseSpider from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): - domain = 'scrapytest.org' + spider = BaseSpider() filter = RequestFingerprintDupeFilter() - filter.open_domain(domain) + filter.open_spider(spider) r1 = Request('http://scrapytest.org/1') r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') - assert not filter.request_seen(domain, r1) - assert filter.request_seen(domain, r1) + assert not filter.request_seen(spider, r1) + assert filter.request_seen(spider, r1) - assert not filter.request_seen(domain, r2) - assert filter.request_seen(domain, r3) + assert not filter.request_seen(spider, r2) + assert filter.request_seen(spider, r3) - filter.close_domain(domain) + filter.close_spider(spider) class NullDupeFilterTest(unittest.TestCase): def test_filter(self): - domain = 'scrapytest.org' + spider = BaseSpider() filter = NullDupeFilter() - filter.open_domain(domain) + filter.open_spider(spider) r1 = Request('http://scrapytest.org/1') - assert not filter.request_seen(domain, r1) - filter.close_domain(domain) + assert not filter.request_seen(spider, r1) + filter.close_spider(spider) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index cfb9c4e2d..db59de2d5 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -1,20 +1,20 @@ import unittest from scrapy.http import Request +from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware -DOMAIN = 'scrapytest.org' - class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = DuplicatesFilterMiddleware() - self.mw.open_domain(DOMAIN) + self.spider = BaseSpider() + self.mw.open_spider(self.spider) def tearDown(self): - self.mw.close_domain(DOMAIN) + self.mw.close_spider(self.spider) def test_process_spider_output(self): @@ -23,7 +23,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): r3 = Request('http://scrapytest.org/2') r4 = Request('http://scrapytest.org/1') - assert not self.mw.enqueue_request(DOMAIN, r1) - assert not self.mw.enqueue_request(DOMAIN, r2) - self.assertRaises(IgnoreRequest, self.mw.enqueue_request, DOMAIN, r3) - self.assertRaises(IgnoreRequest, self.mw.enqueue_request, DOMAIN, r4) + assert not self.mw.enqueue_request(self.spider, r1) + assert not self.mw.enqueue_request(self.spider, r2) + self.assertRaises(IgnoreRequest, self.mw.enqueue_request, self.spider, r3) + self.assertRaises(IgnoreRequest, self.mw.enqueue_request, self.spider, r4)