MemoryUsage: changed .virtual property to methodd. SpiderProfiler: removed dependency on MemoryUsage extension

scrapy/contrib/memusage.py

@@ -27,8 +27,8 @@ class MemoryUsage(object):
             raise NotConfigured
         if not os.path.exists('/proc'):
             raise NotConfigured
-        self.warned = False
 
+        self.warned = False
         self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY')
         self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
         self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
@@ -37,13 +37,11 @@ class MemoryUsage(object):
         dispatcher.connect(self.engine_started, signal=signals.engine_started)
         dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
 
-
-    @property
-    def virtual(self):
+    def get_virtual_size(self):
         return get_vmvalue_from_procfs('VmSize')
 
     def engine_started(self):
-        stats.set_value('memusage/startup', self.virtual)
+        stats.set_value('memusage/startup', self.get_virtual_size())
         self.tasks = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
@@ -63,10 +61,10 @@ class MemoryUsage(object):
                 tsk.stop()
 
     def update(self):
-        stats.max_value('memusage/max', self.virtual)
+        stats.max_value('memusage/max', self.get_virtual_size())
 
     def _check_limit(self):
-        if self.virtual > self.limit:
+        if self.get_virtual_size() > self.limit:
             stats.set_value('memusage/limit_reached', 1)
             mem = self.limit/1024/1024
             log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR)
@@ -80,7 +78,7 @@ class MemoryUsage(object):
     def _check_warning(self):
         if self.warned: # warn only once
             return
-        if self.virtual > self.warning:
+        if self.get_virtual_size() > self.warning:
             stats.set_value('memusage/warning_reached', 1)
             mem = self.warning/1024/1024
             log.msg("Memory usage reached %dM" % mem, level=log.WARNING)
@@ -95,7 +93,7 @@ class MemoryUsage(object):
         """send notification mail with some additional useful info"""
         s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
         s += "Maximum memory usage           : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
-        s += "Current memory usage           : %dM\r\n" % (self.virtual/1024/1024)
+        s += "Current memory usage           : %dM\r\n" % (self.get_virtual_size()/1024/1024)
 
         s += "ENGINE STATUS ------------------------------------------------------- \r\n"
         s += "\r\n"

scrapy/contrib/spider/profiler.py

@@ -13,9 +13,9 @@ from time import time
 
 from scrapy.xlib.pydispatch import dispatcher
 
-from scrapy.extension import extensions
 from scrapy.core import signals
 from scrapy.core.exceptions import NotConfigured
+from scrapy.utils.memory import get_vmvalue_from_procfs
 from scrapy.stats import stats
 from scrapy.conf import settings
 
@@ -24,15 +24,18 @@ class SpiderProfiler(object):
     def __init__(self):
         if not settings.getbool('SPIDERPROFILER_ENABLED'):
             raise NotConfigured
+        try:
+            get_vmvalue_from_procfs('VmSize')
+        except RuntimeError:
+            self._mem_tracking = False
+        else:
+            self._mem_tracking = True
         dispatcher.connect(self._request_received, signals.request_received)
-        dispatcher.connect(self._engine_started, signals.engine_started)
-
-    def _engine_started(self):
-        self.memusage = extensions.enabled.get('MemoryUsage', None)
 
     def _request_received(self, request, spider):
         old_cbs = request.deferred.callbacks[0]
-        new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], old_cbs[0][2]), old_cbs[1])
+        new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], \
+            old_cbs[0][2]), old_cbs[1])
         request.deferred.callbacks[0] = new_cbs
 
     def _profiled_callback(self, function, spider):
@@ -45,16 +48,19 @@ class SpiderProfiler(object):
             domain = spider.domain_name
             tcc = stats.get_value('profiling/total_callback_time', 0, domain=domain)
             sct = stats.get_value('profiling/slowest_callback_time', 0, domain=domain)
-            stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain)
+            stats.set_value('profiling/total_callback_time' % spider.domain_name, \
+                tcc+ct, domain=domain)
             if ct > sct:
                 stats.set_value('profiling/slowest_callback_time', ct, domain=domain)
-                stats.set_value('profiling/slowest_callback_name', function.__name__, domain=domain)
-                stats.set_value('profiling/slowest_callback_url', args[0].url, domain=domain)
-            if self.memusage:
-                tma = stats.get_value('profiling/total_mem_allocated_in_callbacks', 0, domain=domain)
-                stats.set_value('profiling/total_mem_allocated_in_callbacks', tma+mafter-mbefore, domain=domain)
+                stats.set_value('profiling/slowest_callback_name', function.__name__, \
+                    domain=domain)
+                stats.set_value('profiling/slowest_callback_url', args[0].url, \
+                    domain=domain)
+            if self._memusage:
+                stats.inc_value('profiling/total_mem_allocated_in_callbacks', \
+                    count=mafter-mbefore, domain=domain)
             return r
         return new_callback
 
     def _memusage(self):
-        return self.memusage.virtual if self.memusage else 0.0
+        return get_vmvalue_from_procfs('VmSize') if self._mem_tracking else 0.0