diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 905b32d08..01d5d42b0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,7 +4,7 @@ import random from collections import deque from dataclasses import dataclass, field from datetime import datetime -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any from twisted.internet.defer import Deferred, inlineCallbacks @@ -196,7 +196,7 @@ class Downloader: return # Delay queue processing if a download_delay is configured - now = time() + now = monotonic() delay = slot.download_delay() if delay: penalty = delay - now + slot.lastseen @@ -265,7 +265,7 @@ class Downloader: slot.close() def _slot_gc(self, age: float = 60) -> None: - mintime = time() - age + mintime = monotonic() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: self.slots.pop(key).close() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9d102aff8..439638d33 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import logging import re from contextlib import suppress from io import BytesIO -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse @@ -460,7 +460,7 @@ class ScrapyAgent: if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None - start_time = time() + start_time = monotonic() d: Deferred[IResponse] = agent.request( method, to_bytes(url, encoding="ascii"), @@ -489,7 +489,7 @@ class ScrapyAgent: raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return result @staticmethod diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f2774cc07..a4f786363 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,6 +1,6 @@ from __future__ import annotations -from time import time +from time import monotonic from typing import TYPE_CHECKING from urllib.parse import urldefrag @@ -113,7 +113,7 @@ class ScrapyH2Agent: timeout = request.meta.get("download_timeout") or self._connect_timeout agent = self._get_agent(request, timeout) - start_time = time() + start_time = monotonic() d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) @@ -125,7 +125,7 @@ class ScrapyH2Agent: def _cb_latency( response: Response, request: Request, start_time: float ) -> Response: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return response @staticmethod diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 52d287245..2a550cf78 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings -from time import time +from time import monotonic, time from typing import TYPE_CHECKING from urllib.parse import urldefrag, urlparse, urlunparse @@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory): afterFoundGet = False def _build_response(self, body, request): - request.meta["download_latency"] = self.headers_time - self.start_time + request.meta["download_latency"] = ( + self._headers_time_mono - self._start_time_mono + ) status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) @@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.response_headers: Headers | None = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() + self._start_time_mono: float = monotonic() self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( self._build_response, request ) @@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory): def gotHeaders(self, headers): self.headers_time = time() + self._headers_time_mono = monotonic() self.response_headers = headers def gotStatus(self, version, status, message): diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 1400f4c9b..6a5e55992 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times from __future__ import annotations from datetime import datetime, timezone +from time import monotonic from typing import TYPE_CHECKING, Any from scrapy import Spider, signals @@ -21,6 +22,7 @@ class CoreStats: def __init__(self, stats: StatsCollector): self.stats: StatsCollector = stats self.start_time: datetime | None = None + self._start_time_mono: float | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -35,13 +37,14 @@ class CoreStats: def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) + self._start_time_mono = monotonic() self.stats.set_value("start_time", self.start_time) def spider_closed(self, spider: Spider, reason: str) -> None: assert self.start_time is not None - finish_time = datetime.now(tz=timezone.utc) - elapsed_time = finish_time - self.start_time - elapsed_time_seconds = elapsed_time.total_seconds() + assert self._start_time_mono is not None + finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic() + elapsed_time_seconds = finish_time_mono - self._start_time_mono self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds) self.stats.set_value("finish_time", finish_time) self.stats.set_value("finish_reason", reason) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 5b0c66658..ecb4c1492 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -172,7 +172,7 @@ class AsyncioLoopingCall: raise ValueError("Interval must be greater than 0") self.interval = interval - self._start_time = time.time() + self._start_time = time.monotonic() if now: self._call() loop = asyncio.get_event_loop() @@ -182,7 +182,7 @@ class AsyncioLoopingCall: """Return the time to sleep until the next call.""" assert self.interval is not None assert self._start_time is not None - now = time.time() + now = time.monotonic() running_for = now - self._start_time return self.interval - (running_for % self.interval) diff --git a/tests/test_stats.py b/tests/test_stats.py index 2869d302e..27af18bb1 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider: class TestCoreStatsExtension: + @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0) @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector( self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider