diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 4f47342c0..d4be1d091 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -48,6 +48,14 @@ Increment global stat value:: stats.inc_value('spiders_crawled') +Set global stat value only if greater than previous:: + + stats.max_value('max_items_scraped', value, default=0) + +Set global stat value only if lower than previous:: + + stats.min_value('min_free_memory_percent', value, default=100) + Get global stat value:: >>> stats.get_value('spiders_crawled') @@ -64,10 +72,18 @@ signal):: stats.set_value('start_time', datetime.now(), domain='example.com') -Increment domain/spider specific stat value:: +Increment domain-specific stat value:: stats.inc_value('pages_crawled', domain='example.com') +Set domain-specific stat value only if greater than previous:: + + stats.max_value('max_items_scraped', value, default=0, domain='example.com') + +Set domain-specific stat value only if lower than previous:: + + stats.min_value('min_free_memory_percent', value, default=100, domain='example.com') + Get domain-specific stat value:: >>> stats.get_value('pages_crawled', domain='example.com') @@ -131,6 +147,16 @@ class (which they all inherit from). stats table is used, which must be opened or a ``KeyError`` will be raised. + .. method:: max_value(key, value, default, domain=None) + + Set the given value for the given stats only if previous value for same + key (or default if not seted) is lower than value. + + .. method:: min_value(key, value, default, domain=None) + + Set the given value for the given stats only if previous value for same + key (or default if not seted) is greater than value. + .. method:: clear_stats(domain=None) Clear all global stats (if domain is not given) or all domain-specific diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index c145077be..7b0fbcb4b 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -37,6 +37,14 @@ class StatsCollector(object): d = self._stats[domain] d[key] = d.setdefault(key, start) + count + def max_value(self, key, value, default, domain=None): + d = self._stats[domain] + d[key] = max(d.setdefault(key, default), value) + + def min_value(self, key, value, default, domain=None): + d = self._stats[domain] + d[key] = min(d.setdefault(key, default), value) + def clear_stats(self, domain=None): self._stats[domain].clear()