From eae25a04d96877a6d74627492307619d38c96e5c Mon Sep 17 00:00:00 2001 From: Chris Nilsson Date: Sat, 6 Jun 2015 00:39:14 +1000 Subject: [PATCH 1/3] Added MEMUSAGE_CHECK_INTERVAL_SECONDS to Memory usage extension options. Kept the default as it was, at 60.0 seconds. But added a setting to allow this to be changed as desired. --- docs/topics/extensions.rst | 1 + docs/topics/settings.rst | 14 ++++++++++++++ scrapy/extensions/memusage.py | 7 ++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d5d985087..fb5220e9d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -222,6 +222,7 @@ can be configured with the following settings: * :setting:`MEMUSAGE_WARNING_MB` * :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_REPORT` +* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a9eba41ce..b1dee667e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -711,6 +711,20 @@ Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. See :ref:`topics-extensions-ref-memusage`. +.. setting:: MEMUSAGE_CHECK_INTERVAL_SECONDS + +MEMUSAGE_CHECK_INTERVAL_SECONDS +------------------------------- + +Default: ``60.0`` + +Scope: ``scrapy.extensions.memusage`` + +The frequence which the current memory usage will be checked against the +limits set by :setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`. + +See :ref:`topics-extensions-ref-memusage`. + .. setting:: MEMUSAGE_NOTIFY_MAIL MEMUSAGE_NOTIFY_MAIL diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 402c78b68..34bad8246 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -36,6 +36,7 @@ class MemoryUsage(object): self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') + self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS', 60.0) self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -56,15 +57,15 @@ class MemoryUsage(object): self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.limit: tsk = task.LoopingCall(self._check_limit) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) if self.warning: tsk = task.LoopingCall(self._check_warning) self.tasks.append(tsk) - tsk.start(60.0, now=True) + tsk.start(self.check_interval, now=True) def engine_stopped(self): for tsk in self.tasks: From 0c532baf4c76bc902ead3edd2ca0645e1bc66676 Mon Sep 17 00:00:00 2001 From: Chris Nilsson Date: Sat, 6 Jun 2015 11:18:13 +1000 Subject: [PATCH 2/3] Removed typo, and clarified time unit of setting --- docs/topics/settings.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b1dee667e..f89aa30c8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -720,8 +720,12 @@ Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` -The frequence which the current memory usage will be checked against the -limits set by :setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`. +The :ref:`Memory usage extension ` +checks the current memory usage, versus the limits set by +:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, +at fixed time intervals. + +This sets the length of these intervals, in seconds. See :ref:`topics-extensions-ref-memusage`. From 61dec83f706665e5c5238a17f2efdca3d642d9db Mon Sep 17 00:00:00 2001 From: Chris Nilsson Date: Sat, 6 Jun 2015 11:19:29 +1000 Subject: [PATCH 3/3] Moved default value of MEMUSAGE_CHECK_INTERVAL_SECONDS to default_settings --- scrapy/extensions/memusage.py | 2 +- scrapy/settings/default_settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 34bad8246..322213cf0 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -36,7 +36,7 @@ class MemoryUsage(object): self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') - self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS', 60.0) + self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0ea014007..79b6e44f0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -186,6 +186,7 @@ MAIL_USER = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown +MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = False MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = []