diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d5d985087..fb5220e9d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -222,6 +222,7 @@ can be configured with the following settings: * :setting:`MEMUSAGE_WARNING_MB` * :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_REPORT` +* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2b983f9d9..0b41e95c9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -714,6 +714,24 @@ Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. See :ref:`topics-extensions-ref-memusage`. +.. setting:: MEMUSAGE_CHECK_INTERVAL_SECONDS + +MEMUSAGE_CHECK_INTERVAL_SECONDS +------------------------------- + +Default: ``60.0`` + +Scope: ``scrapy.extensions.memusage`` + +The :ref:`Memory usage extension ` +checks the current memory usage, versus the limits set by +:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, +at fixed time intervals. + +This sets the length of these intervals, in seconds. + +See :ref:`topics-extensions-ref-memusage`. + .. setting:: MEMUSAGE_NOTIFY_MAIL MEMUSAGE_NOTIFY_MAIL diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 402c78b68..322213cf0 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -36,6 +36,7 @@ class MemoryUsage(object): self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') + self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -56,15 +57,15 @@ class MemoryUsage(object): self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.limit: tsk = task.LoopingCall(self._check_limit) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.warning: tsk = task.LoopingCall(self._check_warning) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) def engine_stopped(self): for tsk in self.tasks: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 62bb1ccd3..15cd2f413 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -192,6 +192,7 @@ MAIL_USER = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown +MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = False MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = []