RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix the calculate_final_stats method (#6599)

This commit is contained in:
Arthur 2025-01-02 15:36:28 +04:00 committed by GitHub
parent ee239d2451
commit b10d46d280
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/extensions/logstats.py
View File

@ -83,12 +83,15 @@ class LogStats:
self, spider: Spider
) -> tuple[None, None] | tuple[float, float]:
start_time = self.stats.get_value("start_time")
finished_time = self.stats.get_value("finished_time")
finish_time = self.stats.get_value("finish_time")
if not start_time or not finished_time:
if not start_time or not finish_time:
return None, None
mins_elapsed = (finished_time - start_time).seconds / 60
mins_elapsed = (finish_time - start_time).seconds / 60
if mins_elapsed == 0:
return None, None
items = self.stats.get_value("item_scraped_count", 0)
pages = self.stats.get_value("response_received_count", 0)

11
tests/test_logstats.py
View File

@ -47,7 +47,7 @@ class TestLogStats(unittest.TestCase):
# Simulate when spider closes after running for 30 mins
self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972))
self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972))
logstats.spider_closed(self.spider, "test reason")
self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9)
self.assertEqual(self.stats.get_value("items_per_minute"), 116.4)
@ -60,3 +60,12 @@ class TestLogStats(unittest.TestCase):
logstats.spider_closed(self.spider, "test reason")
self.assertIsNone(self.stats.get_value("responses_per_minute"))
self.assertIsNone(self.stats.get_value("items_per_minute"))
def test_stats_calculation_no_elapsed_time(self):
"""The stat values should be None since the elapsed time is 0."""
logstats = LogStats.from_crawler(self.crawler)
self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
logstats.spider_closed(self.spider, "test reason")
self.assertIsNone(self.stats.get_value("responses_per_minute"))
self.assertIsNone(self.stats.get_value("items_per_minute"))