diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d5d985087..fb5220e9d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -222,6 +222,7 @@ can be configured with the following settings: * :setting:`MEMUSAGE_WARNING_MB` * :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_REPORT` +* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a9eba41ce..b1dee667e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -711,6 +711,20 @@ Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. See :ref:`topics-extensions-ref-memusage`. +.. setting:: MEMUSAGE_CHECK_INTERVAL_SECONDS + +MEMUSAGE_CHECK_INTERVAL_SECONDS +------------------------------- + +Default: ``60.0`` + +Scope: ``scrapy.extensions.memusage`` + +The frequence which the current memory usage will be checked against the +limits set by :setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`. + +See :ref:`topics-extensions-ref-memusage`. + .. setting:: MEMUSAGE_NOTIFY_MAIL MEMUSAGE_NOTIFY_MAIL diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 402c78b68..34bad8246 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -36,6 +36,7 @@ class MemoryUsage(object): self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') + self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS', 60.0) self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -56,15 +57,15 @@ class MemoryUsage(object): self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.limit: tsk = task.LoopingCall(self._check_limit) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.warning: tsk = task.LoopingCall(self._check_warning) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) def engine_stopped(self): for tsk in self.tasks: