diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3c4bc662c..2b5770138 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -48,7 +48,8 @@ class Slot(object): if self.closing and not self.inprogress: if self.nextcall: self.nextcall.cancel() - self.heartbeat.stop() + if self.heartbeat.running: + self.heartbeat.stop() self.closing.callback(None) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 647e50f8d..b685e7b19 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -15,6 +15,7 @@ class LogStats(object): self.stats = stats self.interval = interval self.multiplier = 60.0 / self.interval + self.task = None @classmethod def from_crawler(cls, crawler): @@ -47,5 +48,5 @@ class LogStats(object): logger.info(msg, log_args, extra={'spider': spider}) def spider_closed(self, spider, reason): - if self.task.running: + if self.task and self.task.running: self.task.stop() diff --git a/tests/pipelines.py b/tests/pipelines.py new file mode 100644 index 000000000..ddfbc7a99 --- /dev/null +++ b/tests/pipelines.py @@ -0,0 +1,11 @@ +""" +Some pipelines used for testing +""" + +class ZeroDivisionErrorPipeline(object): + + def open_spider(self, spider): + a = 1/0 + + def process_item(self, item, spider): + return item diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 90fd921c8..1b4a4b3b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -250,6 +250,19 @@ with multiples lines yield self.assertFailure(crawler.crawl(), TestError) self.assertFalse(crawler.crawling) + @defer.inlineCallbacks + def test_open_spider_error_on_faulty_pipeline(self): + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) + yield self.assertFailure( + self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + ZeroDivisionError) + self.assertFalse(crawler.crawling) + @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider)