RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MRG+1] Show elapsed time in statscollector (#3638) 

* Update corestats.py

* Update corestats.py

* corrected tests

* Update corestats.py

* Update scrapy/extensions/corestats.py
This commit is contained in:
Maram Sumanth 2019-04-07 00:33:40 +05:30 committed by Mikhail Korobov
parent f816375df5
commit aa46e1995c
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/extensions/corestats.py
View File

@ -24,7 +24,11 @@ class CoreStats(object):
self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
def spider_closed(self, spider, reason):
self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
finish_time = datetime.datetime.utcnow()
elapsed_time = finish_time - self.stats.get_value('start_time')
elapsed_time_seconds = elapsed_time.total_seconds()
self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
self.stats.set_value('finish_time', finish_time, spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):

6
tests/test_closespider.py
View File

@ -53,9 +53,5 @@ class TestCloseSpider(TestCase):
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
reason = crawler.spider.meta['close_reason']
self.assertEqual(reason, 'closespider_timeout')
stats = crawler.stats
start = stats.get_value('start_time')
stop = stats.get_value('finish_time')
diff = stop - start
total_seconds = diff.seconds + diff.microseconds
total_seconds = crawler.stats.get_value('elapsed_time_seconds')
self.assertTrue(total_seconds >= close_on)