RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adding RetryMiddleware errors to the stats 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40489
This commit is contained in:
samus_ 2008-12-14 01:49:59 +00:00
parent bd4c106a61
commit 3d16160bf7
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py
View File

@ -29,6 +29,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
from scrapy import log
from scrapy.core.exceptions import HttpException
from scrapy.stats import stats
from scrapy.conf import settings
class RetryMiddleware(object):
@ -45,6 +46,16 @@ class RetryMiddleware(object):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)):
fp = request.fingerprint()
self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
exception_name = exception.__name__
if exception_name == 'TimeoutError':
if 'defer' in str(exception):
exception_name = 'UserTimeoutError'
else:
exception_name = 'ServerTimeoutError'
stats.incpath('%s/conn_error_count/%s' % (spider.domain_name, exception_name))
else:
stats.incpath('%s/http_error_count/%d' % (spider.domain_name, int(exception.status)))
if self.failed_count[fp] <= self.retry_times:
log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG)