From 479b4942bd88e72485dafda853dedfd494c57414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 3 Oct 2013 12:36:03 -0300 Subject: [PATCH] fix regression on lazy evaluation of start requests --- scrapy/crawler.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 871382eae..9fb630114 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,4 +1,5 @@ import signal +from itertools import chain from twisted.internet import reactor, defer @@ -21,8 +22,7 @@ class Crawler(object): spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) self.spiders = spman_cls.from_crawler(self) - - self.scheduled = {} + self._scheduled = {} def install(self): import scrapy.project @@ -46,20 +46,17 @@ class Crawler(object): def crawl(self, spider, requests=None): spider.set_crawler(self) - if self.configured and self.engine.running: - assert not self.scheduled - return self.schedule(spider, requests) + assert not self._scheduled + return self._schedule(spider, requests) + elif requests is None: + self._scheduled[spider] = None else: - self.scheduled.setdefault(spider, []).append(requests) - - def schedule(self, spider, batches=[]): - requests = [] - for batch in batches: - if batch is None: - batch = spider.start_requests() - requests.extend(batch) + self._scheduled.setdefault(spider, []).append(requests) + def _schedule(self, spider, batches=()): + requests = chain.from_iterable(batches) \ + if batches else spider.start_requests() return self.engine.open_spider(spider, requests) def _spider_closed(self, spider=None): @@ -70,8 +67,8 @@ class Crawler(object): def start(self): yield defer.maybeDeferred(self.configure) - for spider, batches in self.scheduled.iteritems(): - yield self.schedule(spider, batches) + for spider, batches in self._scheduled.iteritems(): + yield self._schedule(spider, batches) yield defer.maybeDeferred(self.engine.start)