RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5722 from JazzGlobal/5717-memusage-logging-extension 

Adds get_virtual_size to _check_limit Logging
This commit is contained in:
Andrey Rahmatullin 2022-11-22 16:08:20 +05:00 committed by GitHub
parent 603e42bc26 eb159c78f1
commit aac6103179
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/extensions/memusage.py
View File

@ -75,7 +75,8 @@ class MemoryUsage:
self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
def _check_limit(self):
if self.get_virtual_size() > self.limit:
current_mem_usage = self.get_virtual_size()
if current_mem_usage > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
@ -92,6 +93,8 @@ class MemoryUsage:
self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
else:
self.crawler.stop()
else:
logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024})
def _check_warning(self):
if self.warned: # warn only once