From b10d46d280fbc84f7a1c50e116a1ed828aa286c9 Mon Sep 17 00:00:00 2001 From: Arthur <48801049+devfox-se@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:36:28 +0400 Subject: [PATCH] Fix the calculate_final_stats method (#6599) --- scrapy/extensions/logstats.py | 9 ++++++--- tests/test_logstats.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index e829d8b92..f2e1f57b8 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -83,12 +83,15 @@ class LogStats: self, spider: Spider ) -> tuple[None, None] | tuple[float, float]: start_time = self.stats.get_value("start_time") - finished_time = self.stats.get_value("finished_time") + finish_time = self.stats.get_value("finish_time") - if not start_time or not finished_time: + if not start_time or not finish_time: return None, None - mins_elapsed = (finished_time - start_time).seconds / 60 + mins_elapsed = (finish_time - start_time).seconds / 60 + + if mins_elapsed == 0: + return None, None items = self.stats.get_value("item_scraped_count", 0) pages = self.stats.get_value("response_received_count", 0) diff --git a/tests/test_logstats.py b/tests/test_logstats.py index d87285df7..a4b002e34 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -47,7 +47,7 @@ class TestLogStats(unittest.TestCase): # Simulate when spider closes after running for 30 mins self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) - self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972)) logstats.spider_closed(self.spider, "test reason") self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) @@ -60,3 +60,12 @@ class TestLogStats(unittest.TestCase): logstats.spider_closed(self.spider, "test reason") self.assertIsNone(self.stats.get_value("responses_per_minute")) self.assertIsNone(self.stats.get_value("items_per_minute")) + + def test_stats_calculation_no_elapsed_time(self): + """The stat values should be None since the elapsed time is 0.""" + logstats = LogStats.from_crawler(self.crawler) + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172)) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute"))