replaced memory usage acounting with (more portable) resource module, removed scrapy.utils.memory module. closes #161

docs/news.rst

@@ -31,6 +31,7 @@ Scrapy changes:
 - promoted :ref:`topics-djangoitem` to main contrib
 - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
 - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
+- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
 
 Scrapyd changes:
 
@@ -377,3 +378,4 @@ First release of Scrapy.
 .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
 .. _lxml: http://lxml.de/
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
+.. _resource: http://docs.python.org/library/resource.html

scrapy/contrib/memusage.py

@@ -12,7 +12,6 @@ from twisted.internet import task
 from scrapy import signals, log
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
-from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported
 from scrapy.utils.engine import get_engine_status
 
 class MemoryUsage(object):
@@ -20,7 +19,9 @@ class MemoryUsage(object):
     def __init__(self, crawler):
         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
             raise NotConfigured
-        if not procfs_supported():
+        try:
+            self.resource = __import__('resource')
+        except ImportError:
             raise NotConfigured
 
         self.crawler = crawler
@@ -38,7 +39,7 @@ class MemoryUsage(object):
         return cls(crawler)
 
     def get_virtual_size(self):
-        return get_vmvalue_from_procfs('VmRSS')
+        return self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss * 1024
 
     def engine_started(self):
         self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())

scrapy/tests/test_utils_memory.py

@@ -1,18 +0,0 @@
-from twisted.trial import unittest
-
-from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported
-
-class UtilsMemoryTestCase(unittest.TestCase):
-
-    def test_get_vmvalue_from_procfs(self):
-        if not procfs_supported():
-            raise unittest.SkipTest('/proc filesystem not supported')
-        vmsize = get_vmvalue_from_procfs('VmSize')
-        vmrss = get_vmvalue_from_procfs('VmRSS')
-        self.assert_(isinstance(vmsize, int))
-        self.assert_(vmsize > 0)
-        self.assert_(vmrss > 0)
-        self.assert_(vmsize > vmrss)
-
-if __name__ == "__main__":
-    unittest.main()

scrapy/utils/memory.py

@@ -1,67 +0,0 @@
-import os
-import sys
-import struct
-
-_vmvalue_scale = {'kB': 1024, 'mB': 1024*1024, 'KB': 1024, 'MB': 1024*1024}
-
-def get_vmvalue_from_procfs(vmkey='VmSize', pid=None):
-    """Return virtual memory value (in bytes) for the given pid using the /proc
-    filesystem. If pid is not given, it default to the current process pid.
-    Available keys are: VmSize, VmRSS (default), VmStk
-    """
-    if pid is None:
-        pid = os.getpid()
-    try:
-        t = open('/proc/%d/status' % pid)
-    except IOError:
-        raise RuntimeError("/proc filesystem not supported")
-    if sys.platform == "sunos5":
-        return _vmvalue_solaris(vmkey, pid)
-    else:
-        v = t.read()
-        t.close()
-        # get vmkey line e.g. 'VmRSS:  9999  kB\n ...'
-        i = v.index(vmkey + ':')
-        v = v[i:].split(None, 3)  # whitespace
-        if len(v) < 3:
-            return 0  # invalid format?
-        # convert Vm value to bytes
-        return int(v[1]) * _vmvalue_scale[v[2]]
-
-def procfs_supported():
-    try:
-        open('/proc/%d/status' % os.getpid())
-    except IOError:
-        return False
-    else:
-        return True
-
-def _vmvalue_solaris(vmkey, pid):
-
-     # Memory layout for struct psinfo.
-     # http://docs.sun.com/app/docs/doc/816-5174/proc-4?l=en&a=view
-    _psinfo_struct_format = (
-        "10i"   # pr_flag [0] through pr_egid [9]
-        "5L"    # pr_addr [10] through pr_ttyydev [14]
-        "2H"    # pr_pctcpu [15] and pr_pctmem [16]
-        "6l"    # pr_start [17-18] through pr_ctime [21-22]
-        "16s"   # pr_fname [23]
-        "80s"   # pr_psargs [24]
-        "2i"    # pr_wstat[25] and pr_argc [26]
-        "2L"    # pr_argv [27] and pr_envp [28]
-        "b3x"   # pr_dmodel[29] and pr_pad2
-        "7i"    # pr_taskid [30] through pr_filler
-        "20i6l" # pr_lwp
-        )
-    psinfo_file = os.path.join("/proc", str(pid), "psinfo")
-    with open(psinfo_file) as f:
-        parts = struct.unpack(_psinfo_struct_format, f.read())
-
-    vmkey_index = {
-        'VmSize' : 11, # pr_size
-        'VmRSS'  : 12, # pr_rssize
-    }
-
-    vm_in_kB = parts[vmkey_index[vmkey]]
-
-    return vm_in_kB * 1024