Changed (unstable) scheduler middleware API to receive spider (instead of domain) in enqueue_request method

docs/experimental/scheduler-middleware.rst

@@ -56,7 +56,7 @@ single Python class that defines one or more of the following methods:
 
 .. class:: SchedulerMiddleware
 
-   .. method:: enqueue_request(domain, request)
+   .. method:: enqueue_request(spider, request)
 
       Process the given request which is being enqueued. This method can return
       None to avoid the request from being scheduled.
@@ -64,8 +64,8 @@ single Python class that defines one or more of the following methods:
       :meth:`enqueue_request` should return either ``None``, a
       :class:`~scrapy.http.Response` object or a ``Deferred``.
 
-      :param domain: the domain originating the request
-      :type domain: string
+      :param spider: the spider originating the request
+      :type spider: :class:`~scrapy.spider.BaseSpider` object
 
       :param requests: the request to be enqueued
       :type request: :class:`~scrapy.http.Request` object

scrapy/contrib/dupefilter.py

@@ -2,13 +2,13 @@
 Dupe Filter classes implement a mechanism for filtering duplicate requests.
 They must implement the following methods:
 
-* open_domain(domain)
-  open a domain for tracking duplicates (typically used to reserve resources)
+* open_spider(spider)
+  open a spider for tracking duplicates (typically used to reserve resources)
 
-* close_domain(domain)
-  close a domain (typically used for freeing resources)
+* close_spider(spider)
+  close a spider (typically used for freeing resources)
 
-* request_seen(domain, request, dont_record=False)
+* request_seen(spider, request, dont_record=False)
   return ``True`` if the request was seen before, or ``False`` otherwise. If
   ``dont_record`` is ``True`` the request must not be recorded as seen.
 
@@ -18,13 +18,13 @@ from scrapy.utils.request import request_fingerprint
 
 
 class NullDupeFilter(dict):
-    def open_domain(self, domain):
+    def open_spider(self, spider):
         pass
 
-    def close_domain(self, domain):
+    def close_spider(self, spider):
         pass
 
-    def request_seen(self, domain, request, dont_record=False):
+    def request_seen(self, spider, request, dont_record=False):
         return False
 
 
@@ -34,16 +34,16 @@ class RequestFingerprintDupeFilter(object):
     def __init__(self):
         self.fingerprints = {}
 
-    def open_domain(self, domain):
-        self.fingerprints[domain] = set()
+    def open_spider(self, spider):
+        self.fingerprints[spider] = set()
 
-    def close_domain(self, domain):
-        del self.fingerprints[domain]
+    def close_spider(self, spider):
+        del self.fingerprints[spider]
 
-    def request_seen(self, domain, request, dont_record=False):
+    def request_seen(self, spider, request, dont_record=False):
         fp = request_fingerprint(request)
-        if fp in self.fingerprints[domain]:
+        if fp in self.fingerprints[spider]:
             return True
         if not dont_record:
-            self.fingerprints[domain].add(fp)
+            self.fingerprints[spider].add(fp)
         return False

scrapy/contrib/schedulermiddleware/duplicatesfilter.py

@@ -17,13 +17,13 @@ class DuplicatesFilterMiddleware(object):
 
         self.dupefilter = load_object(clspath)()
 
-    def enqueue_request(self, domain, request):
-        seen = self.dupefilter.request_seen(domain, request)
+    def enqueue_request(self, spider, request):
+        seen = self.dupefilter.request_seen(spider, request)
         if seen and not request.dont_filter:
             raise IgnoreRequest('Skipped (request already seen)')
 
-    def open_domain(self, domain):
-        self.dupefilter.open_domain(domain)
+    def open_spider(self, spider):
+        self.dupefilter.open_spider(spider)
 
-    def close_domain(self, domain):
-        self.dupefilter.close_domain(domain)
+    def close_spider(self, spider):
+        self.dupefilter.close_spider(spider)

scrapy/core/engine.py

@@ -161,6 +161,7 @@ class ExecutionEngine(object):
 
     def crawl(self, request, spider):
         schd = mustbe_deferred(self.schedule, request, spider)
+        schd.addErrback(log.err, "Unhandled error on engine.crawl()")
         schd.addBoth(self.scraper.enqueue_scrape, request, spider)
         schd.addErrback(log.err, "Unhandled error on engine.crawl()")
         schd.addBoth(lambda _: self.next_request(spider))

scrapy/core/scheduler/middleware.py

@@ -46,7 +46,7 @@ class SchedulerMiddlewareManager(object):
         self.loaded = True
 
     def _add_middleware(self, mw):
-        for name in ['enqueue_request', 'open_domain', 'close_domain']:
+        for name in ['enqueue_request', 'open_spider', 'close_spider']:
             mwfunc = getattr(mw, name, None)
             if mwfunc:
                 self.mw_cbs[name].append(mwfunc)
@@ -54,7 +54,7 @@ class SchedulerMiddlewareManager(object):
     def enqueue_request(self, wrappedfunc, spider, request):
         def _enqueue_request(request):
             for mwfunc in self.mw_cbs['enqueue_request']:
-                result = mwfunc(domain=spider.domain_name, request=request)
+                result = mwfunc(spider=spider, request=request)
                 assert result is None or isinstance(result, (Response, Deferred)), \
                         'Middleware %s.enqueue_request must return None, Response or Deferred, got %s' % \
                         (mwfunc.im_self.__class__.__name__, result.__class__.__name__)
@@ -66,10 +66,10 @@ class SchedulerMiddlewareManager(object):
         return deferred
 
     def open_spider(self, spider):
-        for mwfunc in self.mw_cbs['open_domain']:
-            mwfunc(spider.domain_name)
+        for mwfunc in self.mw_cbs['open_spider']:
+            mwfunc(spider)
 
     def close_spider(self, spider):
-        for mwfunc in self.mw_cbs['close_domain']:
-            mwfunc(spider.domain_name)
+        for mwfunc in self.mw_cbs['close_spider']:
+            mwfunc(spider)
 

scrapy/tests/test_dupefilter.py

@@ -1,36 +1,37 @@
 import unittest
 
 from scrapy.http import Request
+from scrapy.spider import BaseSpider
 from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter
 
 
 class RequestFingerprintDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
-        domain = 'scrapytest.org'
+        spider = BaseSpider()
         filter = RequestFingerprintDupeFilter()
-        filter.open_domain(domain)
+        filter.open_spider(spider)
 
         r1 = Request('http://scrapytest.org/1')
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
 
-        assert not filter.request_seen(domain, r1)
-        assert filter.request_seen(domain, r1)
+        assert not filter.request_seen(spider, r1)
+        assert filter.request_seen(spider, r1)
 
-        assert not filter.request_seen(domain, r2)
-        assert filter.request_seen(domain, r3)
+        assert not filter.request_seen(spider, r2)
+        assert filter.request_seen(spider, r3)
 
-        filter.close_domain(domain)
+        filter.close_spider(spider)
 
 
 class NullDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
-        domain = 'scrapytest.org'
+        spider = BaseSpider()
         filter = NullDupeFilter()
-        filter.open_domain(domain)
+        filter.open_spider(spider)
 
         r1 = Request('http://scrapytest.org/1')
-        assert not filter.request_seen(domain, r1)
-        filter.close_domain(domain)
+        assert not filter.request_seen(spider, r1)
+        filter.close_spider(spider)

scrapy/tests/test_schedulermiddleware_duplicatesfilter.py

@@ -1,20 +1,20 @@
 import unittest
 
 from scrapy.http import Request
+from scrapy.spider import BaseSpider
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
 
 
-DOMAIN = 'scrapytest.org'
-
 class DuplicatesFilterMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
         self.mw = DuplicatesFilterMiddleware()
-        self.mw.open_domain(DOMAIN)
+        self.spider = BaseSpider()
+        self.mw.open_spider(self.spider)
 
     def tearDown(self):
-        self.mw.close_domain(DOMAIN)
+        self.mw.close_spider(self.spider)
 
     def test_process_spider_output(self):
 
@@ -23,7 +23,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
         r3 = Request('http://scrapytest.org/2')
         r4 = Request('http://scrapytest.org/1')
 
-        assert not self.mw.enqueue_request(DOMAIN, r1)
-        assert not self.mw.enqueue_request(DOMAIN, r2)
-        self.assertRaises(IgnoreRequest, self.mw.enqueue_request, DOMAIN, r3)
-        self.assertRaises(IgnoreRequest, self.mw.enqueue_request, DOMAIN, r4)
+        assert not self.mw.enqueue_request(self.spider, r1)
+        assert not self.mw.enqueue_request(self.spider, r2)
+        self.assertRaises(IgnoreRequest, self.mw.enqueue_request, self.spider, r3)
+        self.assertRaises(IgnoreRequest, self.mw.enqueue_request, self.spider, r4)