Sorted out downloader-related stats:

* added DownloaderStats (downloader middleware) to handle all downloader-related stats
 * removed downloader related stats from CoreStats
 * removed downloader exception stats from RetryMiddleware

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40490

scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py

@@ -29,7 +29,6 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
 
 from scrapy import log
 from scrapy.core.exceptions import HttpException
-from scrapy.stats import stats
 from scrapy.conf import settings
 
 class RetryMiddleware(object):
@@ -46,16 +45,6 @@ class RetryMiddleware(object):
         if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)):
             fp = request.fingerprint()
             self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
-            if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
-                exception_name = exception.__name__
-                if exception_name == 'TimeoutError':
-                    if 'defer' in str(exception):
-                        exception_name = 'UserTimeoutError'
-                    else:
-                        exception_name = 'ServerTimeoutError'
-                stats.incpath('%s/conn_error_count/%s' % (spider.domain_name, exception_name))
-            else:
-                stats.incpath('%s/http_error_count/%d' % (spider.domain_name, int(exception.status)))
 
             if self.failed_count[fp] <= self.retry_times:
                 log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG)

scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py

@@ -0,0 +1,49 @@
+from scrapy.core.exceptions import HttpException, NotConfigured
+from scrapy.stats import stats
+from scrapy.conf import settings
+
+class DownloaderStats(object):
+    """DownloaderStats store stats of all requests, responses and
+    exceptions that pass through it.
+
+    They are stored in the following keys:
+
+    SPIDER/downloader/request_method_count
+    SPIDER/downloader/response_status_count
+    SPIDER/downloader/exception_count
+
+    To use this middleware you must enable the DOWNLOADER_STATS setting.
+    """
+
+    def __init__(self):
+        if not settings.getbool('DOWNLOADER_STATS'):
+            raise NotConfigured
+            
+    def process_request(self, request, spider):
+        stats.incpath('_global/downloader/request_count')
+        stats.incpath('%s/downloader/request_count' % spider.domain_name)
+        stats.incpath('%s/downloader/request_method_count/%s' % (spider.domain_name, request.method))
+        reqlen = len(request)
+        stats.incpath('%s/downloader/request_bytes' % spider.domain_name, reqlen)
+        stats.incpath('_global/downloader/request_bytes', reqlen)
+
+    def process_response(self, request, response, spider):
+        self._inc_response_count(response, spider.domain_name)
+        return response
+        
+    def process_exception(self, request, exception, spider):
+        ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
+        stats.incpath('_globl/downloader/exception_count')
+        stats.incpath('%s/downloader/exception_count' % spider.domain_name)
+        stats.incpath('%s/downloader/exception_type_count/%s' % (spider.domain_name, ex_class))
+        if isinstance(exception, HttpException):
+            self._inc_response_count(exception.response, spider.domain_name)
+        raise exception
+
+    def _inc_response_count(self, response, domain):
+        stats.incpath('_global/downloader/response_count')
+        stats.incpath('%s/downloader/response_count' % domain)
+        stats.incpath('%s/downloader/response_status_count/%s' % (domain, response.status))
+        reslen = len(response)
+        stats.incpath('%s/downloader/response_bytes' % domain, reslen)
+        stats.incpath('_global/downloader/response_bytes', reslen)

scrapy/trunk/scrapy/stats/corestats.py

@@ -26,8 +26,6 @@ class CoreStats(object):
         dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
         dispatcher.connect(self.item_passed, signal=signals.item_passed)
         dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
-        dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded)
-        dispatcher.connect(self.request_uploaded, signal=signals.request_uploaded)
 
     def stats_domain_open(self, domain, spider):
         stats.setpath('%s/start_time' % domain, datetime.datetime.now())
@@ -52,18 +50,3 @@ class CoreStats(object):
         stats.incpath('%s/item_dropped_count' % spider.domain_name)
         stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason))
         stats.incpath('_global/item_dropped_count')
-
-    def response_downloaded(self, response, spider):
-        stats.incpath('%s/response_count' % spider.domain_name)
-        stats.incpath('%s/response_status_count/%s' % (spider.domain_name, response.status))
-        stats.incpath('_global/response_downloaded_count')
-
-        reslen = len(response)
-        stats.incpath('%s/transfer/downloaded_bytes' % spider.domain_name, reslen)
-        stats.incpath('_global/transfer/downloaded_bytes', reslen)
-
-    def request_uploaded(self, request, spider):
-        reqlen = len(request)
-        stats.incpath('%s/transfer/uploaded_bytes' % spider.domain_name, reqlen)
-        stats.incpath('_global/transfer/uploaded_bytes', reqlen)
-