core: add missing imports

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40893

scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py

@@ -0,0 +1,18 @@
+"""
+DuplicatesFilterMiddleware: Filter out already visited urls
+"""
+
+from scrapy.http import Request
+from scrapy.conf import settings
+from scrapy.core.exceptions import IgnoreRequest
+from scrapy.core.filters import duplicatesfilter
+from scrapy import log
+
+
+class DuplicatesFilterMiddleware(object):
+    """Filter out already seen requests to avoid visiting pages more than once."""
+
+    def enqueue_request(self, domain, request, priority):
+        added = duplicatesfilter.add(domain, request)
+        if not (added or request.dont_filter):
+            raise IgnoreRequest('Skipped (already seen request')

scrapy/trunk/scrapy/core/scheduler/middleware.py

@@ -0,0 +1,56 @@
+"""
+This module implements the Scheduler Middleware manager.
+
+For more information see the Scheduler Middleware doc in:
+docs/topics/scheduler-middleware.rst
+
+"""
+from twisted.internet.defer import Deferred
+
+from scrapy import log
+from scrapy.http import Request, Response
+from scrapy.core.exceptions import NotConfigured
+from scrapy.utils.misc import load_object
+from scrapy.utils.defer import mustbe_deferred
+from scrapy.conf import settings
+
+class SchedulerMiddlewareManager(object):
+
+    def __init__(self, scheduler):
+        self.loaded = False
+        self.scheduler = scheduler
+        self.mw_enqueue_request = []
+        self.load()
+
+    def _add_middleware(self, mw):
+        if hasattr(mw, 'enqueue_request'):
+            self.mw_enqueue_request.append(mw.enqueue_request)
+
+    def load(self):
+        """Load middleware defined in settings module"""
+        mws = []
+        for mwpath in settings.getlist('SCHEDULER_MIDDLEWARES') or ():
+            cls = load_object(mwpath)
+            if cls:
+                try:
+                    mw = cls()
+                    self._add_middleware(mw)
+                    mws.append(mw)
+                except NotConfigured:
+                    pass
+        log.msg("Enabled scheduler middlewares: %s" % ", ".join([type(m).__name__ for m in mws]))
+        self.loaded = True
+
+    def enqueue_request(self, domain, request, priority):
+        def _enqueue_request(request):
+            for method in self.mw_enqueue_request:
+                result = method(domain=domain, request=request, priority=priority)
+                assert result is None or isinstance(result, (Response, Deferred)), \
+                        'Middleware %s.enqueue_request must return None, Response or Deferred, got %s' % \
+                        (method.im_self.__class__.__name__, result.__class__.__name__)
+                if result:
+                    return result
+            return self.scheduler.enqueue_request(domain=domain, request=request, priority=priority)
+
+        deferred = mustbe_deferred(_enqueue_request, request)
+        return deferred

scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py

@@ -0,0 +1,31 @@
+import unittest
+
+from scrapy.spider import spiders
+from scrapy.http import Request, Response
+from scrapy.core.exceptions import IgnoreRequest
+from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
+from scrapy.core.filters import duplicatesfilter
+
+
+class DuplicatesFilterMiddlewareTest(unittest.TestCase):
+
+    def setUp(self):
+        duplicatesfilter.open('scrapytest.org')
+
+    def tearDown(self):
+        duplicatesfilter.close('scrapytest.org')
+
+    def test_process_spider_output(self):
+        domain = 'scrapytest.org'
+
+        mw = DuplicatesFilterMiddleware()
+
+        r1 = Request('http://scrapytest.org/1')
+        r2 = Request('http://scrapytest.org/2')
+        r3 = Request('http://scrapytest.org/2')
+        r4 = Request('http://scrapytest.org/1')
+
+        assert not mw.enqueue_request(domain, r1, 1)
+        assert not mw.enqueue_request(domain, r2, 1)
+        self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r3, 1)
+        self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r4, 1)