Fixed bugs introduced by previous scheduler middleware change

scrapy/core/engine.py

@@ -197,7 +197,7 @@ class ExecutionEngine(object):
         assert self.has_capacity(), "No free spider slots when opening %r" % \
             spider.name
         log.msg("Spider opened", spider=spider)
-        self.scheduler.open_spider(spider)
+        yield self.scheduler.open_spider(spider)
         self.downloader.open_spider(spider)
         yield self.scraper.open_spider(spider)
         stats.open_spider(spider)

scrapy/core/scheduler.py

@@ -21,7 +21,7 @@ class Scheduler(object):
     def __init__(self):
         self.pending_requests = {}
         self.dfo = settings['SCHEDULER_ORDER'].upper() == 'DFO'
-        self.middleware = SchedulerMiddlewareManager()
+        self.middleware = SchedulerMiddlewareManager.from_settings(settings)
 
     def spider_is_open(self, spider):
         """Check if scheduler's resources were allocated for a spider"""
@@ -39,7 +39,7 @@ class Scheduler(object):
 
         Priority = PriorityStack if self.dfo else PriorityQueue
         self.pending_requests[spider] = Priority()
-        self.middleware.open_spider(spider)
+        return self.middleware.open_spider(spider)
 
     def close_spider(self, spider):
         """Called when a spider has finished scraping to free any resources
@@ -47,8 +47,8 @@ class Scheduler(object):
         """
         if spider not in self.pending_requests:
             raise RuntimeError('Scheduler spider is not open: %s' % spider)
-        self.middleware.close_spider(spider)
         self.pending_requests.pop(spider, None)
+        return self.middleware.close_spider(spider)
 
     def enqueue_request(self, spider, request):
         """Enqueue a request to be downloaded for a spider that is currently being scraped."""

scrapy/core/schedulermw.py

@@ -16,10 +16,11 @@ class SchedulerMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings['DOWNLOADER_MIDDLEWARES_BASE'], \
-            settings['DOWNLOADER_MIDDLEWARES'])
+        return build_component_list(settings['SCHEDULER_MIDDLEWARES_BASE'], \
+            settings['SCHEDULER_MIDDLEWARES'])
 
     def _add_middleware(self, mw):
+        super(SchedulerMiddlewareManager, self)._add_middleware(mw)
         if hasattr(mw, 'enqueue_request'):
             self.methods['enqueue_request'].append(mw.enqueue_request)