From eb159c78f10f546e38e62a9230608269ac13acdd Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 03:36:00 -0500 Subject: [PATCH] Use variable and convert to megabytes. --- scrapy/extensions/memusage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c94899e0f..bf2ee4e6d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,7 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.get_virtual_size() > self.limit: + current_mem_usage = self.get_virtual_size() + if current_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -93,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once