mirror of https://github.com/scrapy/scrapy.git
remove `default` parameter from max_value/min_value stats methods, update docs, and add new methods to dummy collector
This commit is contained in:
parent
9b620652fc
commit
8f16e7f9fa
|
|
@ -50,11 +50,11 @@ Increment global stat value::
|
|||
|
||||
Set global stat value only if greater than previous::
|
||||
|
||||
stats.max_value('max_items_scraped', value, default=0)
|
||||
stats.max_value('max_items_scraped', value)
|
||||
|
||||
Set global stat value only if lower than previous::
|
||||
|
||||
stats.min_value('min_free_memory_percent', value, default=100)
|
||||
stats.min_value('min_free_memory_percent', value)
|
||||
|
||||
Get global stat value::
|
||||
|
||||
|
|
@ -78,11 +78,11 @@ Increment domain-specific stat value::
|
|||
|
||||
Set domain-specific stat value only if greater than previous::
|
||||
|
||||
stats.max_value('max_items_scraped', value, default=0, domain='example.com')
|
||||
stats.max_value('max_items_scraped', value, domain='example.com')
|
||||
|
||||
Set domain-specific stat value only if lower than previous::
|
||||
|
||||
stats.min_value('min_free_memory_percent', value, default=100, domain='example.com')
|
||||
stats.min_value('min_free_memory_percent', value, domain='example.com')
|
||||
|
||||
Get domain-specific stat value::
|
||||
|
||||
|
|
@ -116,12 +116,6 @@ class (which they all inherit from).
|
|||
domain specific one is. If the domain is not yet opened a ``KeyError``
|
||||
exception is raised.
|
||||
|
||||
.. method:: set_value(key, value, domain=None)
|
||||
|
||||
Set the given value for the given stats key. If domain is ``None`` the
|
||||
global stat table is used, otherwise the domain-specific one is, which
|
||||
must be opened or a ``KeyError`` will be raised.
|
||||
|
||||
.. method:: get_stats(domain=None)
|
||||
|
||||
Get all stats from the given domain/spider (if domain is given) or all
|
||||
|
|
@ -147,15 +141,17 @@ class (which they all inherit from).
|
|||
stats table is used, which must be opened or a ``KeyError`` will be
|
||||
raised.
|
||||
|
||||
.. method:: max_value(key, value, default, domain=None)
|
||||
.. method:: max_value(key, value, domain=None)
|
||||
|
||||
Set the given value for the given stats only if previous value for same
|
||||
key (or default if not seted) is lower than value.
|
||||
Set the given value for the given key only if current value for the
|
||||
same key is lower than value. If there is no current value for the
|
||||
given key, the value values is always set.
|
||||
|
||||
.. method:: min_value(key, value, default, domain=None)
|
||||
.. method:: min_value(key, value, domain=None)
|
||||
|
||||
Set the given value for the given stats only if previous value for same
|
||||
key (or default if not seted) is greater than value.
|
||||
Set the given value for the given key only if current value for the
|
||||
same key is greater than value. If there is no current value for the
|
||||
given key, the value values is always set.
|
||||
|
||||
.. method:: clear_stats(domain=None)
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class Scraper(object):
|
|||
site = self.sites[spider.domain_name]
|
||||
dfd = site.add_response_request(response, request)
|
||||
stats.max_value('scraper/max_active_size', \
|
||||
site.active_size, 0, domain=spider.domain_name),
|
||||
site.active_size, domain=spider.domain_name),
|
||||
def finish_scraping(_):
|
||||
site.finish_response(response)
|
||||
self._scrape_next(spider, site)
|
||||
|
|
@ -161,7 +161,7 @@ class Scraper(object):
|
|||
item=output, spider=spider, response=response)
|
||||
self.sites[domain].itemproc_size += 1
|
||||
stats.max_value('scraper/max_itemproc_size', \
|
||||
self.sites[domain].itemproc_size, 0, domain=domain)
|
||||
self.sites[domain].itemproc_size, domain=domain)
|
||||
dfd = self.itemproc.process_item(output, spider)
|
||||
dfd.addBoth(self._itemproc_finished, output, spider)
|
||||
return dfd
|
||||
|
|
|
|||
|
|
@ -37,13 +37,13 @@ class StatsCollector(object):
|
|||
d = self._stats[domain]
|
||||
d[key] = d.setdefault(key, start) + count
|
||||
|
||||
def max_value(self, key, value, default, domain=None):
|
||||
def max_value(self, key, value, domain=None):
|
||||
d = self._stats[domain]
|
||||
d[key] = max(d.setdefault(key, default), value)
|
||||
d[key] = max(d.setdefault(key, value), value)
|
||||
|
||||
def min_value(self, key, value, default, domain=None):
|
||||
def min_value(self, key, value, domain=None):
|
||||
d = self._stats[domain]
|
||||
d[key] = min(d.setdefault(key, default), value)
|
||||
d[key] = min(d.setdefault(key, value), value)
|
||||
|
||||
def clear_stats(self, domain=None):
|
||||
self._stats[domain].clear()
|
||||
|
|
@ -95,3 +95,11 @@ class DummyStatsCollector(StatsCollector):
|
|||
|
||||
def inc_value(self, key, count=1, start=0, domain=None):
|
||||
pass
|
||||
|
||||
def max_value(self, key, value, domain=None):
|
||||
pass
|
||||
|
||||
def min_value(self, key, value, domain=None):
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue