From 495152bd505bccddfe9de7a4fd8585d7815657bb Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 18 May 2011 11:04:48 -0300
Subject: [PATCH] disabled verbose depth stats collection by default, added
 DEPTH_STATS_VERBOSE setting to enable it

---
 docs/topics/settings.rst                    | 12 +++++++++++-
 scrapy/contrib/spidermiddleware/depth.py    | 15 +++++++++------
 scrapy/tests/test_spidermiddleware_depth.py |  2 +-
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 10d80008f..c0df0c69f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -323,7 +323,17 @@ DEPTH_STATS
 
 Default: ``True``
 
-Whether to collect depth stats.
+Whether to collect maximum depth stats.
+
+.. setting:: DEPTH_STATS_VERBOSE
+
+DEPTH_STATS_VERBOSE
+-------------------
+
+Default: ``False``
+
+Whether to collect verbose depth stats. If this is enabled, the number of
+requests for each depth is collected in the stats.
 
 .. setting:: DOWNLOADER_DEBUG
 
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
index 786dd9797..24cd5eba3 100644
--- a/scrapy/contrib/spidermiddleware/depth.py
+++ b/scrapy/contrib/spidermiddleware/depth.py
@@ -9,9 +9,10 @@ from scrapy.http import Request
 
 class DepthMiddleware(object):
 
-    def __init__(self, maxdepth, stats=None):
+    def __init__(self, maxdepth, stats=None, verbose_stats=False):
         self.maxdepth = maxdepth
         self.stats = stats
+        self.verbose_stats = verbose_stats
         if self.stats and self.maxdepth:
             stats.set_value('envinfo/request_depth_limit', maxdepth)
 
@@ -19,11 +20,12 @@ class DepthMiddleware(object):
     def from_settings(cls, settings):
         maxdepth = settings.getint('DEPTH_LIMIT')
         usestats = settings.getbool('DEPTH_STATS')
+        verbose = settings.getbool('DEPTH_STATS_VERBOSE')
         if usestats:
             from scrapy.stats import stats
         else:
             stats = None
-        return cls(maxdepth, stats)
+        return cls(maxdepth, stats, verbose)
 
     def process_spider_output(self, response, result, spider):
         def _filter(request):
@@ -35,14 +37,15 @@ class DepthMiddleware(object):
                         level=log.DEBUG, spider=spider)
                     return False
                 elif self.stats:
-                    self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
-                    if depth > self.stats.get_value('request_depth_max', 0, spider=spider):
-                        self.stats.set_value('request_depth_max', depth, spider=spider)
+                    if self.verbose_stats:
+                        self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
+                    self.stats.max_value('request_depth_max', depth, spider=spider)
             return True
 
         # base case (depth=0)
         if self.stats and 'depth' not in response.request.meta: 
             response.request.meta['depth'] = 0
-            self.stats.inc_value('request_depth_count/0', spider=spider)
+            if self.verbose_stats:
+                self.stats.inc_value('request_depth_count/0', spider=spider)
 
         return (r for r in result or () if _filter(r))
diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/scrapy/tests/test_spidermiddleware_depth.py
index 3cb62ab25..df219cbaa 100644
--- a/scrapy/tests/test_spidermiddleware_depth.py
+++ b/scrapy/tests/test_spidermiddleware_depth.py
@@ -14,7 +14,7 @@ class TestDepthMiddleware(TestCase):
         self.stats = StatsCollector()
         self.stats.open_spider(self.spider)
 
-        self.mw = DepthMiddleware(1, self.stats)
+        self.mw = DepthMiddleware(1, self.stats, True)
         self.assertEquals(self.stats.get_value('envinfo/request_depth_limit'), 1)
 
     def test_process_spider_output(self):