From fe218abbfdf34e10596d3883c4967faa32bf7713 Mon Sep 17 00:00:00 2001 From: Vikas Dhiman Date: Sun, 2 Jan 2011 15:46:11 -0200 Subject: [PATCH] scrapy.utils.memory.get_vmvalue_from_procfs() causes test case failure on SunOS 5.10 i86pc. Modified it to support SunOS 5.10 --- AUTHORS | 1 + scrapy/utils/memory.py | 56 +++++++++++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 74168b0b7..49182f68a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -26,3 +26,4 @@ Here is the list of the primary authors & contributors: * Lucian Ursu * Shuaib Khan * Didier Deshommes + * Vikas Dhiman diff --git a/scrapy/utils/memory.py b/scrapy/utils/memory.py index f2c9a966d..bed6821f9 100644 --- a/scrapy/utils/memory.py +++ b/scrapy/utils/memory.py @@ -1,4 +1,6 @@ import os +import sys +import struct _vmvalue_scale = {'kB': 1024, 'mB': 1024*1024, 'KB': 1024, 'MB': 1024*1024} @@ -13,15 +15,18 @@ def get_vmvalue_from_procfs(vmkey='VmSize', pid=None): t = open('/proc/%d/status' % pid) except IOError: raise RuntimeError("/proc filesystem not supported") - v = t.read() - t.close() - # get vmkey line e.g. 'VmRSS: 9999 kB\n ...' - i = v.index(vmkey + ':') - v = v[i:].split(None, 3) # whitespace - if len(v) < 3: - return 0 # invalid format? - # convert Vm value to bytes - return int(v[1]) * _vmvalue_scale[v[2]] + if sys.platform == "sunos5": + return _vmvalue_solaris(vmkey, pid) + else: + v = t.read() + t.close() + # get vmkey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index(vmkey + ':') + v = v[i:].split(None, 3) # whitespace + if len(v) < 3: + return 0 # invalid format? + # convert Vm value to bytes + return int(v[1]) * _vmvalue_scale[v[2]] def procfs_supported(): try: @@ -30,3 +35,36 @@ def procfs_supported(): return False else: return True + +def _vmvalue_solaris(vmkey, pid): + + # Memory layout for struct psinfo. + # http://docs.sun.com/app/docs/doc/816-5174/proc-4?l=en&a=view + _psinfo_struct_format = ( + "10i" # pr_flag [0] through pr_egid [9] + "5L" # pr_addr [10] through pr_ttyydev [14] + "2H" # pr_pctcpu [15] and pr_pctmem [16] + "6l" # pr_start [17-18] through pr_ctime [21-22] + "16s" # pr_fname [23] + "80s" # pr_psargs [24] + "2i" # pr_wstat[25] and pr_argc [26] + "2L" # pr_argv [27] and pr_envp [28] + "b3x" # pr_dmodel[29] and pr_pad2 + "7i" # pr_taskid [30] through pr_filler + "20i6l" # pr_lwp + ) + psinfo_file = os.path.join("/proc", str(pid), "psinfo") + with open(psinfo_file) as f: + parts = struct.unpack(_psinfo_struct_format, f.read()) + + vmkey_index = { + 'VmSize' : 11, # pr_size + 'VmRSS' : 12, # pr_rssize + } + + vm_in_kB = parts[vmkey_index[vmkey]] + + def kB_to_Bytes(kB): + return kB * 1024 + + return vm_in_kB * 1024