From 7727d87f646d03a9f4d0d30d3dcc1161dd1cb1b5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 7 Nov 2016 16:44:57 +0100
Subject: [PATCH 1/4] Test Slot's heartbeat state before stopping it

Also add a test on state of looping task in LogStats extension

Fixes #2011 and #2362
---
 scrapy/core/engine.py         | 3 ++-
 scrapy/extensions/logstats.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 3c4bc662c..2b5770138 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -48,7 +48,8 @@ class Slot(object):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
-                self.heartbeat.stop()
+                if self.heartbeat.running:
+                    self.heartbeat.stop()
             self.closing.callback(None)
 
 
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 647e50f8d..b685e7b19 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -15,6 +15,7 @@ class LogStats(object):
         self.stats = stats
         self.interval = interval
         self.multiplier = 60.0 / self.interval
+        self.task = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -47,5 +48,5 @@ class LogStats(object):
         logger.info(msg, log_args, extra={'spider': spider})
 
     def spider_closed(self, spider, reason):
-        if self.task.running:
+        if self.task and self.task.running:
             self.task.stop()

From 61efacdd1f6fb55cf1f694e56502a3d8a1d5a325 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 11:35:42 +0100
Subject: [PATCH 2/4] Add testcase for catching exception from open_spider()
 from pipeline

---
 tests/pipelines.py  | 11 +++++++++++
 tests/test_crawl.py | 12 ++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 tests/pipelines.py

diff --git a/tests/pipelines.py b/tests/pipelines.py
new file mode 100644
index 000000000..d81cfa93d
--- /dev/null
+++ b/tests/pipelines.py
@@ -0,0 +1,11 @@
+"""
+Some pipelines used for testing and benchmarking
+"""
+
+class ZeroDivisionErrorPipeline(object):
+
+    def open_spider(self, spider):
+        a = 1/0
+
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 90fd921c8..a0f3c9997 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -250,6 +250,18 @@ with multiples lines
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
 
+    @defer.inlineCallbacks
+    def test_open_spider_error_on_faulty_pipeline(self):
+        settings = {
+            "ITEM_PIPELINES": {
+                "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+            }
+        }
+        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        yield self.assertFailure(
+            self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+            ZeroDivisionError)
+
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
         crawler = self.runner.create_crawler(SimpleSpider)

From 27456996a904ba25f780d2e3824059e836b6cb90 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 11:46:16 +0100
Subject: [PATCH 3/4] Add assertion on crawler not running

---
 tests/test_crawl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index a0f3c9997..1b4a4b3b0 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -261,6 +261,7 @@ with multiples lines
         yield self.assertFailure(
             self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
             ZeroDivisionError)
+        self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):

From af2280e695f9430fdb0ba452594f3f9c4cb9592d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 13:30:51 +0100
Subject: [PATCH 4/4] Update docstring

---
 tests/pipelines.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pipelines.py b/tests/pipelines.py
index d81cfa93d..ddfbc7a99 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -1,5 +1,5 @@
 """
-Some pipelines used for testing and benchmarking
+Some pipelines used for testing
 """
 
 class ZeroDivisionErrorPipeline(object):