Merge pull request #1282 from otherchirps/memusage-check-interval

[MRG+1] Added MEMUSAGE_CHECK_INTERVAL_SECONDS to Memory usage extension options.

docs/topics/extensions.rst

@@ -222,6 +222,7 @@ can be configured with the following settings:
 * :setting:`MEMUSAGE_WARNING_MB`
 * :setting:`MEMUSAGE_NOTIFY_MAIL`
 * :setting:`MEMUSAGE_REPORT`
+* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
 
 Memory debugger extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~

docs/topics/settings.rst

@@ -714,6 +714,24 @@ Scrapy  (if MEMUSAGE_ENABLED is True). If zero, no check will be performed.
 
 See :ref:`topics-extensions-ref-memusage`.
 
+.. setting:: MEMUSAGE_CHECK_INTERVAL_SECONDS
+
+MEMUSAGE_CHECK_INTERVAL_SECONDS
+-------------------------------
+
+Default: ``60.0``
+
+Scope: ``scrapy.extensions.memusage``
+
+The :ref:`Memory usage extension <topics-extensions-ref-memusage>`
+checks the current memory usage, versus the limits set by 
+:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, 
+at fixed time intervals.
+
+This sets the length of these intervals, in seconds.
+
+See :ref:`topics-extensions-ref-memusage`.
+
 .. setting:: MEMUSAGE_NOTIFY_MAIL
 
 MEMUSAGE_NOTIFY_MAIL

scrapy/extensions/memusage.py

@@ -36,6 +36,7 @@ class MemoryUsage(object):
         self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
         self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
         self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
+        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@@ -56,15 +57,15 @@ class MemoryUsage(object):
         self.tasks = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
-        tsk.start(60.0, now=True)
+        tsk.start(self.check_interval, now=True)
         if self.limit:
             tsk = task.LoopingCall(self._check_limit)
             self.tasks.append(tsk)
-            tsk.start(60.0, now=True)
+            tsk.start(self.check_interval, now=True)
         if self.warning:
             tsk = task.LoopingCall(self._check_warning)
             self.tasks.append(tsk)
-            tsk.start(60.0, now=True)
+            tsk.start(self.check_interval, now=True)
 
     def engine_stopped(self):
         for tsk in self.tasks:

scrapy/settings/default_settings.py

@@ -192,6 +192,7 @@ MAIL_USER = None
 MEMDEBUG_ENABLED = False        # enable memory debugging
 MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
 
+MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
 MEMUSAGE_ENABLED = False
 MEMUSAGE_LIMIT_MB = 0
 MEMUSAGE_NOTIFY_MAIL = []