mirror of https://github.com/scrapy/scrapy.git
stats collector gains two new methods to store values only if greater/lower than previous value
This commit is contained in:
parent
9a531b552e
commit
cd8eed77c5
|
|
@ -48,6 +48,14 @@ Increment global stat value::
|
|||
|
||||
stats.inc_value('spiders_crawled')
|
||||
|
||||
Set global stat value only if greater than previous::
|
||||
|
||||
stats.max_value('max_items_scraped', value, default=0)
|
||||
|
||||
Set global stat value only if lower than previous::
|
||||
|
||||
stats.min_value('min_free_memory_percent', value, default=100)
|
||||
|
||||
Get global stat value::
|
||||
|
||||
>>> stats.get_value('spiders_crawled')
|
||||
|
|
@ -64,10 +72,18 @@ signal)::
|
|||
|
||||
stats.set_value('start_time', datetime.now(), domain='example.com')
|
||||
|
||||
Increment domain/spider specific stat value::
|
||||
Increment domain-specific stat value::
|
||||
|
||||
stats.inc_value('pages_crawled', domain='example.com')
|
||||
|
||||
Set domain-specific stat value only if greater than previous::
|
||||
|
||||
stats.max_value('max_items_scraped', value, default=0, domain='example.com')
|
||||
|
||||
Set domain-specific stat value only if lower than previous::
|
||||
|
||||
stats.min_value('min_free_memory_percent', value, default=100, domain='example.com')
|
||||
|
||||
Get domain-specific stat value::
|
||||
|
||||
>>> stats.get_value('pages_crawled', domain='example.com')
|
||||
|
|
@ -131,6 +147,16 @@ class (which they all inherit from).
|
|||
stats table is used, which must be opened or a ``KeyError`` will be
|
||||
raised.
|
||||
|
||||
.. method:: max_value(key, value, default, domain=None)
|
||||
|
||||
Set the given value for the given stats only if previous value for same
|
||||
key (or default if not seted) is lower than value.
|
||||
|
||||
.. method:: min_value(key, value, default, domain=None)
|
||||
|
||||
Set the given value for the given stats only if previous value for same
|
||||
key (or default if not seted) is greater than value.
|
||||
|
||||
.. method:: clear_stats(domain=None)
|
||||
|
||||
Clear all global stats (if domain is not given) or all domain-specific
|
||||
|
|
|
|||
|
|
@ -37,6 +37,14 @@ class StatsCollector(object):
|
|||
d = self._stats[domain]
|
||||
d[key] = d.setdefault(key, start) + count
|
||||
|
||||
def max_value(self, key, value, default, domain=None):
|
||||
d = self._stats[domain]
|
||||
d[key] = max(d.setdefault(key, default), value)
|
||||
|
||||
def min_value(self, key, value, default, domain=None):
|
||||
d = self._stats[domain]
|
||||
d[key] = min(d.setdefault(key, default), value)
|
||||
|
||||
def clear_stats(self, domain=None):
|
||||
self._stats[domain].clear()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue