From 8f16e7f9fa17f987f8ac36c0da9a2bd47de45e66 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 11 Aug 2009 16:23:23 -0300 Subject: [PATCH] remove `default` parameter from max_value/min_value stats methods, update docs, and add new methods to dummy collector --- docs/topics/stats.rst | 28 ++++++++++++---------------- scrapy/core/scraper.py | 4 ++-- scrapy/stats/collector/__init__.py | 16 ++++++++++++---- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index d4be1d091..2ee580b8a 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -50,11 +50,11 @@ Increment global stat value:: Set global stat value only if greater than previous:: - stats.max_value('max_items_scraped', value, default=0) + stats.max_value('max_items_scraped', value) Set global stat value only if lower than previous:: - stats.min_value('min_free_memory_percent', value, default=100) + stats.min_value('min_free_memory_percent', value) Get global stat value:: @@ -78,11 +78,11 @@ Increment domain-specific stat value:: Set domain-specific stat value only if greater than previous:: - stats.max_value('max_items_scraped', value, default=0, domain='example.com') + stats.max_value('max_items_scraped', value, domain='example.com') Set domain-specific stat value only if lower than previous:: - stats.min_value('min_free_memory_percent', value, default=100, domain='example.com') + stats.min_value('min_free_memory_percent', value, domain='example.com') Get domain-specific stat value:: @@ -116,12 +116,6 @@ class (which they all inherit from). domain specific one is. If the domain is not yet opened a ``KeyError`` exception is raised. - .. method:: set_value(key, value, domain=None) - - Set the given value for the given stats key. If domain is ``None`` the - global stat table is used, otherwise the domain-specific one is, which - must be opened or a ``KeyError`` will be raised. - .. method:: get_stats(domain=None) Get all stats from the given domain/spider (if domain is given) or all @@ -147,15 +141,17 @@ class (which they all inherit from). stats table is used, which must be opened or a ``KeyError`` will be raised. - .. method:: max_value(key, value, default, domain=None) + .. method:: max_value(key, value, domain=None) - Set the given value for the given stats only if previous value for same - key (or default if not seted) is lower than value. + Set the given value for the given key only if current value for the + same key is lower than value. If there is no current value for the + given key, the value values is always set. - .. method:: min_value(key, value, default, domain=None) + .. method:: min_value(key, value, domain=None) - Set the given value for the given stats only if previous value for same - key (or default if not seted) is greater than value. + Set the given value for the given key only if current value for the + same key is greater than value. If there is no current value for the + given key, the value values is always set. .. method:: clear_stats(domain=None) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0a1ecd10e..8ed1d92e1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -85,7 +85,7 @@ class Scraper(object): site = self.sites[spider.domain_name] dfd = site.add_response_request(response, request) stats.max_value('scraper/max_active_size', \ - site.active_size, 0, domain=spider.domain_name), + site.active_size, domain=spider.domain_name), def finish_scraping(_): site.finish_response(response) self._scrape_next(spider, site) @@ -161,7 +161,7 @@ class Scraper(object): item=output, spider=spider, response=response) self.sites[domain].itemproc_size += 1 stats.max_value('scraper/max_itemproc_size', \ - self.sites[domain].itemproc_size, 0, domain=domain) + self.sites[domain].itemproc_size, domain=domain) dfd = self.itemproc.process_item(output, spider) dfd.addBoth(self._itemproc_finished, output, spider) return dfd diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index 7b0fbcb4b..7fe1553af 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -37,13 +37,13 @@ class StatsCollector(object): d = self._stats[domain] d[key] = d.setdefault(key, start) + count - def max_value(self, key, value, default, domain=None): + def max_value(self, key, value, domain=None): d = self._stats[domain] - d[key] = max(d.setdefault(key, default), value) + d[key] = max(d.setdefault(key, value), value) - def min_value(self, key, value, default, domain=None): + def min_value(self, key, value, domain=None): d = self._stats[domain] - d[key] = min(d.setdefault(key, default), value) + d[key] = min(d.setdefault(key, value), value) def clear_stats(self, domain=None): self._stats[domain].clear() @@ -95,3 +95,11 @@ class DummyStatsCollector(StatsCollector): def inc_value(self, key, count=1, start=0, domain=None): pass + + def max_value(self, key, value, domain=None): + pass + + def min_value(self, key, value, domain=None): + pass + +