Merge pull request #2382 from redapple/slot-heartbeat-state-test

Test Slot's heartbeat state before stopping it

scrapy/core/engine.py

@@ -48,7 +48,8 @@ class Slot(object):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
-                self.heartbeat.stop()
+                if self.heartbeat.running:
+                    self.heartbeat.stop()
             self.closing.callback(None)
 
 

scrapy/extensions/logstats.py

@@ -15,6 +15,7 @@ class LogStats(object):
         self.stats = stats
         self.interval = interval
         self.multiplier = 60.0 / self.interval
+        self.task = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -47,5 +48,5 @@ class LogStats(object):
         logger.info(msg, log_args, extra={'spider': spider})
 
     def spider_closed(self, spider, reason):
-        if self.task.running:
+        if self.task and self.task.running:
             self.task.stop()

tests/pipelines.py

@@ -0,0 +1,11 @@
+"""
+Some pipelines used for testing
+"""
+
+class ZeroDivisionErrorPipeline(object):
+
+    def open_spider(self, spider):
+        a = 1/0
+
+    def process_item(self, item, spider):
+        return item

tests/test_crawl.py

@@ -250,6 +250,19 @@ with multiples lines
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
 
+    @defer.inlineCallbacks
+    def test_open_spider_error_on_faulty_pipeline(self):
+        settings = {
+            "ITEM_PIPELINES": {
+                "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+            }
+        }
+        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        yield self.assertFailure(
+            self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+            ZeroDivisionError)
+        self.assertFalse(crawler.crawling)
+
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
         crawler = self.runner.create_crawler(SimpleSpider)