From 8ba6fa2096346d08a3481cee52e583333e7384b2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 7 Sep 2010 17:47:47 -0300 Subject: [PATCH] Fixed bugs introduced by previous scheduler middleware change --- scrapy/core/engine.py | 2 +- scrapy/core/scheduler.py | 6 +++--- scrapy/core/schedulermw.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3370b2c30..17a683f7f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -197,7 +197,7 @@ class ExecutionEngine(object): assert self.has_capacity(), "No free spider slots when opening %r" % \ spider.name log.msg("Spider opened", spider=spider) - self.scheduler.open_spider(spider) + yield self.scheduler.open_spider(spider) self.downloader.open_spider(spider) yield self.scraper.open_spider(spider) stats.open_spider(spider) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 2da44cbdd..93987cfba 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -21,7 +21,7 @@ class Scheduler(object): def __init__(self): self.pending_requests = {} self.dfo = settings['SCHEDULER_ORDER'].upper() == 'DFO' - self.middleware = SchedulerMiddlewareManager() + self.middleware = SchedulerMiddlewareManager.from_settings(settings) def spider_is_open(self, spider): """Check if scheduler's resources were allocated for a spider""" @@ -39,7 +39,7 @@ class Scheduler(object): Priority = PriorityStack if self.dfo else PriorityQueue self.pending_requests[spider] = Priority() - self.middleware.open_spider(spider) + return self.middleware.open_spider(spider) def close_spider(self, spider): """Called when a spider has finished scraping to free any resources @@ -47,8 +47,8 @@ class Scheduler(object): """ if spider not in self.pending_requests: raise RuntimeError('Scheduler spider is not open: %s' % spider) - self.middleware.close_spider(spider) self.pending_requests.pop(spider, None) + return self.middleware.close_spider(spider) def enqueue_request(self, spider, request): """Enqueue a request to be downloaded for a spider that is currently being scraped.""" diff --git a/scrapy/core/schedulermw.py b/scrapy/core/schedulermw.py index dca0476d1..4ea7f3e0b 100644 --- a/scrapy/core/schedulermw.py +++ b/scrapy/core/schedulermw.py @@ -16,10 +16,11 @@ class SchedulerMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings['DOWNLOADER_MIDDLEWARES_BASE'], \ - settings['DOWNLOADER_MIDDLEWARES']) + return build_component_list(settings['SCHEDULER_MIDDLEWARES_BASE'], \ + settings['SCHEDULER_MIDDLEWARES']) def _add_middleware(self, mw): + super(SchedulerMiddlewareManager, self)._add_middleware(mw) if hasattr(mw, 'enqueue_request'): self.methods['enqueue_request'].append(mw.enqueue_request)