From aa46e1995cd5cb1099aba17535372b538bd656b3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Sun, 7 Apr 2019 00:33:40 +0530 Subject: [PATCH] [MRG+1] Show elapsed time in statscollector (#3638) * Update corestats.py * Update corestats.py * corrected tests * Update corestats.py * Update scrapy/extensions/corestats.py --- scrapy/extensions/corestats.py | 6 +++++- tests/test_closespider.py | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 3d9a307b7..8cc5e18ac 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -24,7 +24,11 @@ class CoreStats(object): self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) def spider_closed(self, spider, reason): - self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) + finish_time = datetime.datetime.utcnow() + elapsed_time = finish_time - self.stats.get_value('start_time') + elapsed_time_seconds = elapsed_time.total_seconds() + self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider) + self.stats.set_value('finish_time', finish_time, spider=spider) self.stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 0eb1b7944..4a56425b7 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -53,9 +53,5 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') - stats = crawler.stats - start = stats.get_value('start_time') - stop = stats.get_value('finish_time') - diff = stop - start - total_seconds = diff.seconds + diff.microseconds + total_seconds = crawler.stats.get_value('elapsed_time_seconds') self.assertTrue(total_seconds >= close_on)