CloseSpider extension: Added support for closing spider after N errors have been raised. Closes #254

scrapy/contrib/closespider.py

@@ -7,9 +7,10 @@ See documentation in docs/topics/extensions.rst
 from collections import defaultdict
 
 from twisted.internet import reactor
+from twisted.python import log as txlog
 from scrapy.xlib.pydispatch import dispatcher
 
-from scrapy import signals
+from scrapy import signals, log
 from scrapy.project import crawler
 from scrapy.conf import settings
 
@@ -19,11 +20,15 @@ class CloseSpider(object):
         self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
         self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED')
         self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT')
+        self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT')
 
+        self.errorcounts = defaultdict(int)
         self.pagecounts = defaultdict(int)
         self.counts = defaultdict(int)
         self.tasks = {}
 
+        if self.errorcount:
+            txlog.addObserver(self.catch_log)
         if self.pagecount:
             dispatcher.connect(self.page_count, signal=signals.response_received)
         if self.timeout:
@@ -32,6 +37,14 @@ class CloseSpider(object):
             dispatcher.connect(self.item_passed, signal=signals.item_passed)
         dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
 
+    def catch_log(self, event):
+        if event.get('logLevel') == log.ERROR:
+            spider = event.get('spider')
+            if spider:
+                self.errorcounts[spider] += 1
+                if self.errorcounts[spider] == self.errorcount:
+                    crawler.engine.close_spider(spider, 'closespider_errorcount')
+
     def page_count(self, response, request, spider):
         self.pagecounts[spider] += 1
         if self.pagecounts[spider] == self.pagecount:
@@ -49,6 +62,8 @@ class CloseSpider(object):
 
     def spider_closed(self, spider):
         self.counts.pop(spider, None)
+        self.pagecounts.pop(spider, None)
+        self.errorcounts.pop(spider, None)
         tsk = self.tasks.pop(spider, None)
         if tsk and tsk.active():
             tsk.cancel()