From c9f3decddd84d956fdc604754e1e8dc89ced1938 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 1 Jul 2015 04:50:21 -0300
Subject: [PATCH] Use psutil if available to measure RSS on memusage extension

---
 scrapy/extensions/memusage.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 402c78b68..29d6cc26a 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -24,10 +24,9 @@ class MemoryUsage(object):
     def __init__(self, crawler):
         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
             raise NotConfigured
-        try:
-            # stdlib's resource module is only available on unix platforms.
-            self.resource = import_module('resource')
-        except ImportError:
+
+        self._get_rss = self._choose_rss_func()
+        if self._get_rss is None:
             raise NotConfigured
 
         self.crawler = crawler
@@ -45,12 +44,32 @@ class MemoryUsage(object):
         return cls(crawler)
 
     def get_virtual_size(self):
+        return self._get_rss()
+
+    def _get_rss_psutil(self):
+        return self.psutil_module.Process().memory_info()[0]
+
+    def _get_rss_resource(self):
         size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
         if sys.platform != 'darwin':
             # on Mac OS X ru_maxrss is in bytes, on Linux it is in KB
             size *= 1024
         return size
 
+    def _choose_rss_func(self):
+        try:
+            # psutil is recommended as it can measure current RSS usage
+            self.psutil_module = import_module('psutil')
+            return self._get_rss_psutil
+        except ImportError:
+            pass
+        try:
+            # stdlib's resource module is only available on unix platforms.
+            self.resource = import_module('resource')
+            return self._get_rss_resource
+        except ImportError:
+            pass
+
     def engine_started(self):
         self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
         self.tasks = []
@@ -96,6 +115,7 @@ class MemoryUsage(object):
     def _check_warning(self):
         if self.warned: # warn only once
             return
+
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
             mem = self.warning/1024/1024