diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index 0a04309cf..49fe6ecb7 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -7,9 +7,10 @@ See documentation in docs/topics/extensions.rst from collections import defaultdict from twisted.internet import reactor +from twisted.python import log as txlog from scrapy.xlib.pydispatch import dispatcher -from scrapy import signals +from scrapy import signals, log from scrapy.project import crawler from scrapy.conf import settings @@ -19,11 +20,15 @@ class CloseSpider(object): self.timeout = settings.getint('CLOSESPIDER_TIMEOUT') self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED') self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT') + self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT') + self.errorcounts = defaultdict(int) self.pagecounts = defaultdict(int) self.counts = defaultdict(int) self.tasks = {} + if self.errorcount: + txlog.addObserver(self.catch_log) if self.pagecount: dispatcher.connect(self.page_count, signal=signals.response_received) if self.timeout: @@ -32,6 +37,14 @@ class CloseSpider(object): dispatcher.connect(self.item_passed, signal=signals.item_passed) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + def catch_log(self, event): + if event.get('logLevel') == log.ERROR: + spider = event.get('spider') + if spider: + self.errorcounts[spider] += 1 + if self.errorcounts[spider] == self.errorcount: + crawler.engine.close_spider(spider, 'closespider_errorcount') + def page_count(self, response, request, spider): self.pagecounts[spider] += 1 if self.pagecounts[spider] == self.pagecount: @@ -49,6 +62,8 @@ class CloseSpider(object): def spider_closed(self, spider): self.counts.pop(spider, None) + self.pagecounts.pop(spider, None) + self.errorcounts.pop(spider, None) tsk = self.tasks.pop(spider, None) if tsk and tsk.active(): tsk.cancel()