From 24a18e9af13a482c1dd6036046226cf57318477b Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Mon, 21 Nov 2022 17:41:06 -0500 Subject: [PATCH 1/3] Adds virtualsize property to _check_limit error log. --- scrapy/extensions/memusage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index f5081a7d7..aba0c8d7e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem}, extra={'crawler': self.crawler}) + {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " From 8f2adad7a7f2d858acd082afd6e9a6835bfddcd7 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 00:48:24 -0500 Subject: [PATCH 2/3] Log self.get_virtual_size() on every call of _check_limit --- scrapy/extensions/memusage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index aba0c8d7e..c94899e0f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) + {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " @@ -92,6 +92,8 @@ class MemoryUsage: self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() + else: + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) def _check_warning(self): if self.warned: # warn only once From eb159c78f10f546e38e62a9230608269ac13acdd Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 03:36:00 -0500 Subject: [PATCH 3/3] Use variable and convert to megabytes. --- scrapy/extensions/memusage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c94899e0f..bf2ee4e6d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,7 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.get_virtual_size() > self.limit: + current_mem_usage = self.get_virtual_size() + if current_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -93,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once