From dd80f6acdf3d181a158013077893258003ef72e8 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 31 Aug 2009 12:14:30 -0300 Subject: [PATCH] MemoryUsage: changed .virtual property to methodd. SpiderProfiler: removed dependency on MemoryUsage extension --- scrapy/contrib/memusage.py | 16 +++++++--------- scrapy/contrib/spider/profiler.py | 32 ++++++++++++++++++------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index df7ef7fea..196e4d2dd 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -27,8 +27,8 @@ class MemoryUsage(object): raise NotConfigured if not os.path.exists('/proc'): raise NotConfigured - self.warned = False + self.warned = False self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY') self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 @@ -37,13 +37,11 @@ class MemoryUsage(object): dispatcher.connect(self.engine_started, signal=signals.engine_started) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) - - @property - def virtual(self): + def get_virtual_size(self): return get_vmvalue_from_procfs('VmSize') def engine_started(self): - stats.set_value('memusage/startup', self.virtual) + stats.set_value('memusage/startup', self.get_virtual_size()) self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) @@ -63,10 +61,10 @@ class MemoryUsage(object): tsk.stop() def update(self): - stats.max_value('memusage/max', self.virtual) + stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.virtual > self.limit: + if self.get_virtual_size() > self.limit: stats.set_value('memusage/limit_reached', 1) mem = self.limit/1024/1024 log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) @@ -80,7 +78,7 @@ class MemoryUsage(object): def _check_warning(self): if self.warned: # warn only once return - if self.virtual > self.warning: + if self.get_virtual_size() > self.warning: stats.set_value('memusage/warning_reached', 1) mem = self.warning/1024/1024 log.msg("Memory usage reached %dM" % mem, level=log.WARNING) @@ -95,7 +93,7 @@ class MemoryUsage(object): """send notification mail with some additional useful info""" s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.virtual/1024/1024) + s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" diff --git a/scrapy/contrib/spider/profiler.py b/scrapy/contrib/spider/profiler.py index 0f2d9bb9e..b7c3ace7c 100644 --- a/scrapy/contrib/spider/profiler.py +++ b/scrapy/contrib/spider/profiler.py @@ -13,9 +13,9 @@ from time import time from scrapy.xlib.pydispatch import dispatcher -from scrapy.extension import extensions from scrapy.core import signals from scrapy.core.exceptions import NotConfigured +from scrapy.utils.memory import get_vmvalue_from_procfs from scrapy.stats import stats from scrapy.conf import settings @@ -24,15 +24,18 @@ class SpiderProfiler(object): def __init__(self): if not settings.getbool('SPIDERPROFILER_ENABLED'): raise NotConfigured + try: + get_vmvalue_from_procfs('VmSize') + except RuntimeError: + self._mem_tracking = False + else: + self._mem_tracking = True dispatcher.connect(self._request_received, signals.request_received) - dispatcher.connect(self._engine_started, signals.engine_started) - - def _engine_started(self): - self.memusage = extensions.enabled.get('MemoryUsage', None) def _request_received(self, request, spider): old_cbs = request.deferred.callbacks[0] - new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], old_cbs[0][2]), old_cbs[1]) + new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], \ + old_cbs[0][2]), old_cbs[1]) request.deferred.callbacks[0] = new_cbs def _profiled_callback(self, function, spider): @@ -45,16 +48,19 @@ class SpiderProfiler(object): domain = spider.domain_name tcc = stats.get_value('profiling/total_callback_time', 0, domain=domain) sct = stats.get_value('profiling/slowest_callback_time', 0, domain=domain) - stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain) + stats.set_value('profiling/total_callback_time' % spider.domain_name, \ + tcc+ct, domain=domain) if ct > sct: stats.set_value('profiling/slowest_callback_time', ct, domain=domain) - stats.set_value('profiling/slowest_callback_name', function.__name__, domain=domain) - stats.set_value('profiling/slowest_callback_url', args[0].url, domain=domain) - if self.memusage: - tma = stats.get_value('profiling/total_mem_allocated_in_callbacks', 0, domain=domain) - stats.set_value('profiling/total_mem_allocated_in_callbacks', tma+mafter-mbefore, domain=domain) + stats.set_value('profiling/slowest_callback_name', function.__name__, \ + domain=domain) + stats.set_value('profiling/slowest_callback_url', args[0].url, \ + domain=domain) + if self._memusage: + stats.inc_value('profiling/total_mem_allocated_in_callbacks', \ + count=mafter-mbefore, domain=domain) return r return new_callback def _memusage(self): - return self.memusage.virtual if self.memusage else 0.0 + return get_vmvalue_from_procfs('VmSize') if self._mem_tracking else 0.0