From 3d16160bf7626dfc94f4000d20e2d5f74c362eb3 Mon Sep 17 00:00:00 2001 From: samus_ Date: Sun, 14 Dec 2008 01:49:59 +0000 Subject: [PATCH] adding RetryMiddleware errors to the stats --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40489 --- .../scrapy/contrib/downloadermiddleware/retry.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py index e7ed17ea5..6af0c3f3b 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -29,6 +29,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError from scrapy import log from scrapy.core.exceptions import HttpException +from scrapy.stats import stats from scrapy.conf import settings class RetryMiddleware(object): @@ -45,6 +46,16 @@ class RetryMiddleware(object): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)): fp = request.fingerprint() self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 + if isinstance(exception, self.EXCEPTIONS_TO_RETRY): + exception_name = exception.__name__ + if exception_name == 'TimeoutError': + if 'defer' in str(exception): + exception_name = 'UserTimeoutError' + else: + exception_name = 'ServerTimeoutError' + stats.incpath('%s/conn_error_count/%s' % (spider.domain_name, exception_name)) + else: + stats.incpath('%s/http_error_count/%d' % (spider.domain_name, int(exception.status))) if self.failed_count[fp] <= self.retry_times: log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG)