remove `default` parameter from max_value/min_value stats methods, update docs, and add new methods to dummy collector

This commit is contained in:
Daniel Grana 2009-08-11 16:23:23 -03:00
parent 9b620652fc
commit 8f16e7f9fa
3 changed files with 26 additions and 22 deletions

View File

@ -50,11 +50,11 @@ Increment global stat value::
Set global stat value only if greater than previous::
stats.max_value('max_items_scraped', value, default=0)
stats.max_value('max_items_scraped', value)
Set global stat value only if lower than previous::
stats.min_value('min_free_memory_percent', value, default=100)
stats.min_value('min_free_memory_percent', value)
Get global stat value::
@ -78,11 +78,11 @@ Increment domain-specific stat value::
Set domain-specific stat value only if greater than previous::
stats.max_value('max_items_scraped', value, default=0, domain='example.com')
stats.max_value('max_items_scraped', value, domain='example.com')
Set domain-specific stat value only if lower than previous::
stats.min_value('min_free_memory_percent', value, default=100, domain='example.com')
stats.min_value('min_free_memory_percent', value, domain='example.com')
Get domain-specific stat value::
@ -116,12 +116,6 @@ class (which they all inherit from).
domain specific one is. If the domain is not yet opened a ``KeyError``
exception is raised.
.. method:: set_value(key, value, domain=None)
Set the given value for the given stats key. If domain is ``None`` the
global stat table is used, otherwise the domain-specific one is, which
must be opened or a ``KeyError`` will be raised.
.. method:: get_stats(domain=None)
Get all stats from the given domain/spider (if domain is given) or all
@ -147,15 +141,17 @@ class (which they all inherit from).
stats table is used, which must be opened or a ``KeyError`` will be
raised.
.. method:: max_value(key, value, default, domain=None)
.. method:: max_value(key, value, domain=None)
Set the given value for the given stats only if previous value for same
key (or default if not seted) is lower than value.
Set the given value for the given key only if current value for the
same key is lower than value. If there is no current value for the
given key, the value values is always set.
.. method:: min_value(key, value, default, domain=None)
.. method:: min_value(key, value, domain=None)
Set the given value for the given stats only if previous value for same
key (or default if not seted) is greater than value.
Set the given value for the given key only if current value for the
same key is greater than value. If there is no current value for the
given key, the value values is always set.
.. method:: clear_stats(domain=None)

View File

@ -85,7 +85,7 @@ class Scraper(object):
site = self.sites[spider.domain_name]
dfd = site.add_response_request(response, request)
stats.max_value('scraper/max_active_size', \
site.active_size, 0, domain=spider.domain_name),
site.active_size, domain=spider.domain_name),
def finish_scraping(_):
site.finish_response(response)
self._scrape_next(spider, site)
@ -161,7 +161,7 @@ class Scraper(object):
item=output, spider=spider, response=response)
self.sites[domain].itemproc_size += 1
stats.max_value('scraper/max_itemproc_size', \
self.sites[domain].itemproc_size, 0, domain=domain)
self.sites[domain].itemproc_size, domain=domain)
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, spider)
return dfd

View File

@ -37,13 +37,13 @@ class StatsCollector(object):
d = self._stats[domain]
d[key] = d.setdefault(key, start) + count
def max_value(self, key, value, default, domain=None):
def max_value(self, key, value, domain=None):
d = self._stats[domain]
d[key] = max(d.setdefault(key, default), value)
d[key] = max(d.setdefault(key, value), value)
def min_value(self, key, value, default, domain=None):
def min_value(self, key, value, domain=None):
d = self._stats[domain]
d[key] = min(d.setdefault(key, default), value)
d[key] = min(d.setdefault(key, value), value)
def clear_stats(self, domain=None):
self._stats[domain].clear()
@ -95,3 +95,11 @@ class DummyStatsCollector(StatsCollector):
def inc_value(self, key, count=1, start=0, domain=None):
pass
def max_value(self, key, value, domain=None):
pass
def min_value(self, key, value, domain=None):
pass