Improve handling of time deltas (#7377)

scrapy/core/downloader/__init__.py

@@ -4,7 +4,7 @@ import random
 from collections import deque
 from dataclasses import dataclass, field
 from datetime import datetime
-from time import time
+from time import monotonic
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -196,7 +196,7 @@ class Downloader:
             return
 
         # Delay queue processing if a download_delay is configured
-        now = time()
+        now = monotonic()
         delay = slot.download_delay()
         if delay:
             penalty = delay - now + slot.lastseen
@@ -265,7 +265,7 @@ class Downloader:
             slot.close()
 
     def _slot_gc(self, age: float = 60) -> None:
-        mintime = time() - age
+        mintime = monotonic() - age
         for key, slot in list(self.slots.items()):
             if not slot.active and slot.lastseen + slot.delay < mintime:
                 self.slots.pop(key).close()

scrapy/core/downloader/handlers/http11.py

@@ -7,7 +7,7 @@ import logging
 import re
 from contextlib import suppress
 from io import BytesIO
-from time import time
+from time import monotonic
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
 from urllib.parse import urldefrag, urlparse
 
@@ -460,7 +460,7 @@ class ScrapyAgent:
         if isinstance(agent, self._TunnelingAgent):
             headers.removeHeader(b"Proxy-Authorization")
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
-        start_time = time()
+        start_time = monotonic()
         d: Deferred[IResponse] = agent.request(
             method,
             to_bytes(url, encoding="ascii"),
@@ -489,7 +489,7 @@ class ScrapyAgent:
         raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
 
     def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
-        request.meta["download_latency"] = time() - start_time
+        request.meta["download_latency"] = monotonic() - start_time
         return result
 
     @staticmethod

scrapy/core/downloader/handlers/http2.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from time import time
+from time import monotonic
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag
 
@@ -113,7 +113,7 @@ class ScrapyH2Agent:
         timeout = request.meta.get("download_timeout") or self._connect_timeout
         agent = self._get_agent(request, timeout)
 
-        start_time = time()
+        start_time = monotonic()
         d = agent.request(request, spider)
         d.addCallback(self._cb_latency, request, start_time)
 
@@ -125,7 +125,7 @@ class ScrapyH2Agent:
     def _cb_latency(
         response: Response, request: Request, start_time: float
     ) -> Response:
-        request.meta["download_latency"] = time() - start_time
+        request.meta["download_latency"] = monotonic() - start_time
         return response
 
     @staticmethod

scrapy/core/downloader/webclient.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import warnings
-from time import time
+from time import monotonic, time
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag, urlparse, urlunparse
 
@@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory):
     afterFoundGet = False
 
     def _build_response(self, body, request):
-        request.meta["download_latency"] = self.headers_time - self.start_time
+        request.meta["download_latency"] = (
+            self._headers_time_mono - self._start_time_mono
+        )
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
@@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         self.response_headers: Headers | None = None
         self.timeout: float = request.meta.get("download_timeout") or timeout
         self.start_time: float = time()
+        self._start_time_mono: float = monotonic()
         self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
             self._build_response, request
         )
@@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
 
     def gotHeaders(self, headers):
         self.headers_time = time()
+        self._headers_time_mono = monotonic()
         self.response_headers = headers
 
     def gotStatus(self, version, status, message):

scrapy/extensions/corestats.py

@@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times
 from __future__ import annotations
 
 from datetime import datetime, timezone
+from time import monotonic
 from typing import TYPE_CHECKING, Any
 
 from scrapy import Spider, signals
@@ -21,6 +22,7 @@ class CoreStats:
     def __init__(self, stats: StatsCollector):
         self.stats: StatsCollector = stats
         self.start_time: datetime | None = None
+        self._start_time_mono: float | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
@@ -35,13 +37,14 @@ class CoreStats:
 
     def spider_opened(self, spider: Spider) -> None:
         self.start_time = datetime.now(tz=timezone.utc)
+        self._start_time_mono = monotonic()
         self.stats.set_value("start_time", self.start_time)
 
     def spider_closed(self, spider: Spider, reason: str) -> None:
         assert self.start_time is not None
-        finish_time = datetime.now(tz=timezone.utc)
-        elapsed_time = finish_time - self.start_time
-        elapsed_time_seconds = elapsed_time.total_seconds()
+        assert self._start_time_mono is not None
+        finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic()
+        elapsed_time_seconds = finish_time_mono - self._start_time_mono
         self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)
         self.stats.set_value("finish_time", finish_time)
         self.stats.set_value("finish_reason", reason)

scrapy/utils/asyncio.py

@@ -172,7 +172,7 @@ class AsyncioLoopingCall:
             raise ValueError("Interval must be greater than 0")
 
         self.interval = interval
-        self._start_time = time.time()
+        self._start_time = time.monotonic()
         if now:
             self._call()
         loop = asyncio.get_event_loop()
@@ -182,7 +182,7 @@ class AsyncioLoopingCall:
         """Return the time to sleep until the next call."""
         assert self.interval is not None
         assert self._start_time is not None
-        now = time.time()
+        now = time.monotonic()
         running_for = now - self._start_time
         return self.interval - (running_for % self.interval)
 

tests/test_stats.py

@@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider:
 
 
 class TestCoreStatsExtension:
+    @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
     @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(
         self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider