diff --git a/docs/news.rst b/docs/news.rst index e03e7206f..c454efcc2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -31,6 +31,7 @@ Scrapy changes: - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor +- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module Scrapyd changes: @@ -377,3 +378,4 @@ First release of Scrapy. .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _lxml: http://lxml.de/ .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ +.. _resource: http://docs.python.org/library/resource.html diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 191bacb73..8f315b4c7 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -12,7 +12,6 @@ from twisted.internet import task from scrapy import signals, log from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported from scrapy.utils.engine import get_engine_status class MemoryUsage(object): @@ -20,7 +19,9 @@ class MemoryUsage(object): def __init__(self, crawler): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): raise NotConfigured - if not procfs_supported(): + try: + self.resource = __import__('resource') + except ImportError: raise NotConfigured self.crawler = crawler @@ -38,7 +39,7 @@ class MemoryUsage(object): return cls(crawler) def get_virtual_size(self): - return get_vmvalue_from_procfs('VmRSS') + return self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss * 1024 def engine_started(self): self.crawler.stats.set_value('memusage/startup', self.get_virtual_size()) diff --git a/scrapy/tests/test_utils_memory.py b/scrapy/tests/test_utils_memory.py deleted file mode 100644 index c5557d573..000000000 --- a/scrapy/tests/test_utils_memory.py +++ /dev/null @@ -1,18 +0,0 @@ -from twisted.trial import unittest - -from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported - -class UtilsMemoryTestCase(unittest.TestCase): - - def test_get_vmvalue_from_procfs(self): - if not procfs_supported(): - raise unittest.SkipTest('/proc filesystem not supported') - vmsize = get_vmvalue_from_procfs('VmSize') - vmrss = get_vmvalue_from_procfs('VmRSS') - self.assert_(isinstance(vmsize, int)) - self.assert_(vmsize > 0) - self.assert_(vmrss > 0) - self.assert_(vmsize > vmrss) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/utils/memory.py b/scrapy/utils/memory.py deleted file mode 100644 index aad0c2935..000000000 --- a/scrapy/utils/memory.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import sys -import struct - -_vmvalue_scale = {'kB': 1024, 'mB': 1024*1024, 'KB': 1024, 'MB': 1024*1024} - -def get_vmvalue_from_procfs(vmkey='VmSize', pid=None): - """Return virtual memory value (in bytes) for the given pid using the /proc - filesystem. If pid is not given, it default to the current process pid. - Available keys are: VmSize, VmRSS (default), VmStk - """ - if pid is None: - pid = os.getpid() - try: - t = open('/proc/%d/status' % pid) - except IOError: - raise RuntimeError("/proc filesystem not supported") - if sys.platform == "sunos5": - return _vmvalue_solaris(vmkey, pid) - else: - v = t.read() - t.close() - # get vmkey line e.g. 'VmRSS: 9999 kB\n ...' - i = v.index(vmkey + ':') - v = v[i:].split(None, 3) # whitespace - if len(v) < 3: - return 0 # invalid format? - # convert Vm value to bytes - return int(v[1]) * _vmvalue_scale[v[2]] - -def procfs_supported(): - try: - open('/proc/%d/status' % os.getpid()) - except IOError: - return False - else: - return True - -def _vmvalue_solaris(vmkey, pid): - - # Memory layout for struct psinfo. - # http://docs.sun.com/app/docs/doc/816-5174/proc-4?l=en&a=view - _psinfo_struct_format = ( - "10i" # pr_flag [0] through pr_egid [9] - "5L" # pr_addr [10] through pr_ttyydev [14] - "2H" # pr_pctcpu [15] and pr_pctmem [16] - "6l" # pr_start [17-18] through pr_ctime [21-22] - "16s" # pr_fname [23] - "80s" # pr_psargs [24] - "2i" # pr_wstat[25] and pr_argc [26] - "2L" # pr_argv [27] and pr_envp [28] - "b3x" # pr_dmodel[29] and pr_pad2 - "7i" # pr_taskid [30] through pr_filler - "20i6l" # pr_lwp - ) - psinfo_file = os.path.join("/proc", str(pid), "psinfo") - with open(psinfo_file) as f: - parts = struct.unpack(_psinfo_struct_format, f.read()) - - vmkey_index = { - 'VmSize' : 11, # pr_size - 'VmRSS' : 12, # pr_rssize - } - - vm_in_kB = parts[vmkey_index[vmkey]] - - return vm_in_kB * 1024