diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py index 6af0c3f3b..e7ed17ea5 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -29,7 +29,6 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError from scrapy import log from scrapy.core.exceptions import HttpException -from scrapy.stats import stats from scrapy.conf import settings class RetryMiddleware(object): @@ -46,16 +45,6 @@ class RetryMiddleware(object): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)): fp = request.fingerprint() self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - exception_name = exception.__name__ - if exception_name == 'TimeoutError': - if 'defer' in str(exception): - exception_name = 'UserTimeoutError' - else: - exception_name = 'ServerTimeoutError' - stats.incpath('%s/conn_error_count/%s' % (spider.domain_name, exception_name)) - else: - stats.incpath('%s/http_error_count/%d' % (spider.domain_name, int(exception.status))) if self.failed_count[fp] <= self.retry_times: log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py new file mode 100644 index 000000000..9fe0b866a --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py @@ -0,0 +1,49 @@ +from scrapy.core.exceptions import HttpException, NotConfigured +from scrapy.stats import stats +from scrapy.conf import settings + +class DownloaderStats(object): + """DownloaderStats store stats of all requests, responses and + exceptions that pass through it. + + They are stored in the following keys: + + SPIDER/downloader/request_method_count + SPIDER/downloader/response_status_count + SPIDER/downloader/exception_count + + To use this middleware you must enable the DOWNLOADER_STATS setting. + """ + + def __init__(self): + if not settings.getbool('DOWNLOADER_STATS'): + raise NotConfigured + + def process_request(self, request, spider): + stats.incpath('_global/downloader/request_count') + stats.incpath('%s/downloader/request_count' % spider.domain_name) + stats.incpath('%s/downloader/request_method_count/%s' % (spider.domain_name, request.method)) + reqlen = len(request) + stats.incpath('%s/downloader/request_bytes' % spider.domain_name, reqlen) + stats.incpath('_global/downloader/request_bytes', reqlen) + + def process_response(self, request, response, spider): + self._inc_response_count(response, spider.domain_name) + return response + + def process_exception(self, request, exception, spider): + ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) + stats.incpath('_globl/downloader/exception_count') + stats.incpath('%s/downloader/exception_count' % spider.domain_name) + stats.incpath('%s/downloader/exception_type_count/%s' % (spider.domain_name, ex_class)) + if isinstance(exception, HttpException): + self._inc_response_count(exception.response, spider.domain_name) + raise exception + + def _inc_response_count(self, response, domain): + stats.incpath('_global/downloader/response_count') + stats.incpath('%s/downloader/response_count' % domain) + stats.incpath('%s/downloader/response_status_count/%s' % (domain, response.status)) + reslen = len(response) + stats.incpath('%s/downloader/response_bytes' % domain, reslen) + stats.incpath('_global/downloader/response_bytes', reslen) diff --git a/scrapy/trunk/scrapy/stats/corestats.py b/scrapy/trunk/scrapy/stats/corestats.py index 20435a04c..079d8dab8 100644 --- a/scrapy/trunk/scrapy/stats/corestats.py +++ b/scrapy/trunk/scrapy/stats/corestats.py @@ -26,8 +26,6 @@ class CoreStats(object): dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.item_passed, signal=signals.item_passed) dispatcher.connect(self.item_dropped, signal=signals.item_dropped) - dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) - dispatcher.connect(self.request_uploaded, signal=signals.request_uploaded) def stats_domain_open(self, domain, spider): stats.setpath('%s/start_time' % domain, datetime.datetime.now()) @@ -52,18 +50,3 @@ class CoreStats(object): stats.incpath('%s/item_dropped_count' % spider.domain_name) stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason)) stats.incpath('_global/item_dropped_count') - - def response_downloaded(self, response, spider): - stats.incpath('%s/response_count' % spider.domain_name) - stats.incpath('%s/response_status_count/%s' % (spider.domain_name, response.status)) - stats.incpath('_global/response_downloaded_count') - - reslen = len(response) - stats.incpath('%s/transfer/downloaded_bytes' % spider.domain_name, reslen) - stats.incpath('_global/transfer/downloaded_bytes', reslen) - - def request_uploaded(self, request, spider): - reqlen = len(request) - stats.incpath('%s/transfer/uploaded_bytes' % spider.domain_name, reqlen) - stats.incpath('_global/transfer/uploaded_bytes', reqlen) -