From 8f2adad7a7f2d858acd082afd6e9a6835bfddcd7 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 00:48:24 -0500 Subject: [PATCH] Log self.get_virtual_size() on every call of _check_limit --- scrapy/extensions/memusage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index aba0c8d7e..c94899e0f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) + {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " @@ -92,6 +92,8 @@ class MemoryUsage: self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() + else: + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) def _check_warning(self): if self.warned: # warn only once