From 7fe7c3f3b1f35dac692e61ef92783beb689462fe Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 23 Feb 2012 16:55:06 -0200 Subject: [PATCH] MemoryUsage extension: close the spiders (instead of stopping the engine) when the limit is exceeded, providing a descriptive reason for the close. Also fixed default value of MEMUSAGE_ENABLED setting to match the documentation. --- docs/topics/extensions.rst | 10 +++++----- scrapy/contrib/memusage.py | 7 ++++++- scrapy/settings/default_settings.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d00bfdc01..212043988 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -240,15 +240,15 @@ Memory usage extension .. note:: This extension does not work in Windows. -Allows monitoring the memory used by a Scrapy process and: +Monitors the memory used by the Scrapy process that runs the spider and: -1, send a notification e-mail when it exceeds a certain value -2. terminate the Scrapy process when it exceeds a certain value +1, sends a notification e-mail when it exceeds a certain value +2. closes the spider when it exceeds a certain value The notification e-mails can be triggered when a certain warning value is reached (:setting:`MEMUSAGE_WARNING_MB`) and when the maximum value is reached -(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the Scrapy process to be -terminated. +(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the spider to be closed +and the Scrapy process to be terminated. This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and can be configured with the following settings: diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index fe3fa0e9e..0eda342ab 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -76,7 +76,12 @@ class MemoryUsage(object): (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) stats.set_value('memusage/limit_notified', 1) - self.crawler.stop() + open_spiders = self.crawler.engine.open_spiders + if open_spiders: + for spider in open_spiders: + self.crawler.engine.close_spider(spider, 'memusage_exceeded') + else: + self.crawler.stop() def _check_warning(self): if self.warned: # warn only once diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 278ce8570..5b5216ca1 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -203,7 +203,7 @@ MAIL_USER = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown -MEMUSAGE_ENABLED = 1 +MEMUSAGE_ENABLED = False MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_REPORT = False