From 184166d62958ae087cfed1ba489ab28982312bea Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 29 Jun 2026 10:58:18 +0200 Subject: [PATCH] Use get_* for getters --- scrapy/core/engine.py | 6 +++--- scrapy/core/scheduler.py | 6 +++--- scrapy/pqueues.py | 8 ++++---- scrapy/throttling.py | 20 ++++++++++---------- tests/test_engine.py | 10 +++++----- tests/test_pqueues.py | 20 ++++++++++---------- tests/test_scheduler.py | 4 ++-- tests/test_throttling.py | 24 ++++++++++++------------ 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 042468cd7..75a70ce9a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -140,7 +140,7 @@ class ExecutionEngine: self._scheduling: int = 0 # A coalesced wakeup timer, armed when a throttling-aware scheduler # reports that all pending requests are time-blocked (see - # ``next_request_delay``). + # ``get_next_request_delay``). self._throttling_wakeup: CallLaterResult | None = None self._delayed_requests_warn_threshold: int = self.settings.getint( "DELAYED_REQUESTS_WARN_THRESHOLD" @@ -376,7 +376,7 @@ class ExecutionEngine: """ assert self._slot is not None scheduler = self._slot.scheduler - delay_fn = getattr(scheduler, "next_request_delay", None) + delay_fn = getattr(scheduler, "get_next_request_delay", None) if delay_fn is None or not scheduler.has_pending_requests(): return delay = delay_fn() @@ -433,7 +433,7 @@ class ExecutionEngine: # scheduler instead of in _throttling_waiting, so it does not hit this # path; recommend it only when it is not already in use. scheduler = self._slot.scheduler if self._slot is not None else None - if scheduler is None or not hasattr(scheduler, "next_request_delay"): + if scheduler is None or not hasattr(scheduler, "get_next_request_delay"): recommendation = ( " Consider switching to scrapy.core.scheduler.ThrottlingAwareScheduler." ) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 76eb3b5fe..90a892dc3 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -544,7 +544,7 @@ class ThrottlingAwareScheduler(Scheduler): def open(self, spider: Spider) -> Deferred[None] | None: result = super().open(spider) - if not hasattr(self.mqs, "next_request_delay"): + if not hasattr(self.mqs, "get_next_request_delay"): raise ValueError( f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be " f"set to a throttling-aware priority queue such as " @@ -619,7 +619,7 @@ class ThrottlingAwareScheduler(Scheduler): return False return True - def next_request_delay(self) -> float | None: + def get_next_request_delay(self) -> float | None: """Return the minimum number of seconds until some pending request becomes sendable because a time-based throttling gate opens, or ``None`` if no pending request is time-blocked. @@ -630,6 +630,6 @@ class ThrottlingAwareScheduler(Scheduler): delays = [ delay for pq in (self.mqs, self.dqs) - if pq is not None and (delay := pq.next_request_delay()) is not None # type: ignore[attr-defined] + if pq is not None and (delay := pq.get_next_request_delay()) is not None # type: ignore[attr-defined] ] return min(delays) if delays else None diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 5843c9e9e..dfee687b4 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -603,7 +603,7 @@ class ThrottlingAwarePriorityQueue: Among the sendable queues (those whose scope set can be sent right now), the one whose head has the highest request priority is chosen; ties are broken by ascending load (the maximum - :meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the + :meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scope_load` over the scopes of the queue), i.e. by preferring the least-busy scopes. """ self._promote_ready(time.monotonic()) @@ -614,7 +614,7 @@ class ThrottlingAwarePriorityQueue: if head is None or not self._throttler.is_ready(head): continue load = max( - (self._throttler.scope_load(scope_id) for scope_id in scope_set), + (self._throttler.get_scope_load(scope_id) for scope_id in scope_set), default=0.0, ) sort_key = (queue.priority(head), load) @@ -641,7 +641,7 @@ class ThrottlingAwarePriorityQueue: return None return selected[1].peek() - def next_request_delay(self) -> float | None: + def get_next_request_delay(self) -> float | None: now = time.monotonic() self._promote_ready(now) delay: float | None = None @@ -651,7 +651,7 @@ class ThrottlingAwarePriorityQueue: continue if self._throttler.is_ready(head): return 0.0 - head_delay = self._throttler.time_until_ready(head) + head_delay = self._throttler.get_time_until_ready(head) if head_delay is None: continue if delay is None or head_delay < delay: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index ddb540f53..ebb1a903e 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -342,7 +342,7 @@ class ThrottlingManagerProtocol(Protocol): returned ``True``). The reservation is released by :meth:`release`. """ - def time_until_ready(self, request: Request) -> float | None: + def get_time_until_ready(self, request: Request) -> float | None: """Return the number of seconds until every time-based gate of *request* would be open, or ``None`` if no time-based gate is currently blocking it (only a concurrency slot could be). @@ -352,7 +352,7 @@ class ThrottlingManagerProtocol(Protocol): requests are time-blocked. """ - def scope_load(self, scope_id: str) -> float: + def get_scope_load(self, scope_id: str) -> float: """Return the current load of the scope identified by *scope_id*: its active sends divided by its concurrency limit (or by the global :setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit). @@ -510,7 +510,7 @@ class ThrottlingManager: It backs :meth:`get_scopes` and is also the fallback for the synchronous readiness methods (:meth:`is_ready`, :meth:`reserve`, - :meth:`time_until_ready`) when no scopes were persisted on + :meth:`get_time_until_ready`) when no scopes were persisted on ``request.meta`` by an earlier :meth:`get_scopes` call (which normally happens at enqueue time and survives disk restores; see :func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be @@ -712,7 +712,7 @@ class ThrottlingManager: manager.record_sent(amount=value) self._reserved[request] = managers - def time_until_ready(self, request: Request) -> float | None: + def get_time_until_ready(self, request: Request) -> float | None: now = time.monotonic() wait = max(0.0, self._request_delay_deadline(request, now) - now) for scope_id, value in self._cached_scope_values(request): @@ -720,8 +720,8 @@ class ThrottlingManager: wait = max(wait, manager.can_send(now=now, amount=value)) return wait if wait > 0 else None - def scope_load(self, scope_id: ScopeID) -> float: - return self._get_scope_manager(scope_id).load() + def get_scope_load(self, scope_id: ScopeID) -> float: + return self._get_scope_manager(scope_id).get_load() def get_request_delay(self, request: Request, now: float | None = None) -> float: now = time.monotonic() if now is None else now @@ -760,7 +760,7 @@ class ThrottlingManager: This is the readiness-API counterpart of :meth:`_delay_request`: a throttling-aware scheduler gates requests through :meth:`is_ready` and - :meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the + :meth:`get_time_until_ready` instead of awaiting :meth:`acquire`, so the delay is enforced by holding back the request until this deadline rather than by sleeping. The deadline is computed once, the first time the request reaches the gate, and stored so later polls reuse it. @@ -996,7 +996,7 @@ class ThrottlingScopeManagerProtocol(Protocol): Return ``False`` when no concurrency limit is enforced. """ - def load(self) -> float: + def get_load(self) -> float: """Return the current load of this scope: a non-negative number, with ``1.0`` meaning "as busy as its concurrency limit allows". @@ -1123,7 +1123,7 @@ class ThrottlingScopeManager: else: self._concurrency = None # Used as the load denominator when the scope enforces no explicit - # concurrency limit (see load()). + # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") # Quota. @@ -1281,7 +1281,7 @@ class ThrottlingScopeManager: def concurrency_blocked(self) -> bool: return self._concurrency is not None and self._active >= self._concurrency - def load(self) -> float: + def get_load(self) -> float: limit = ( self._concurrency if self._concurrency is not None diff --git a/tests/test_engine.py b/tests/test_engine.py index 032e5df2b..2b15c40d1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -677,7 +677,7 @@ class TestEngineThrottling: def test_maybe_arm_throttling_wakeup_arms_timer(self, engine): scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = 5.0 + scheduler.get_next_request_delay.return_value = 5.0 engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -688,7 +688,7 @@ class TestEngineThrottling: def test_maybe_arm_throttling_wakeup_no_delay(self, engine): scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = None + scheduler.get_next_request_delay.return_value = None engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -700,7 +700,7 @@ class TestEngineThrottling: # the engine, so no timer must be armed. scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = 0.0 + scheduler.get_next_request_delay.return_value = 0.0 engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -710,7 +710,7 @@ class TestEngineThrottling: engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} engine._slot = Mock() - # A scheduler without next_request_delay is not throttling-aware, so the + # A scheduler without get_next_request_delay is not throttling-aware, so the # warning recommends switching to one. engine._slot.scheduler = Mock(spec=BaseScheduler) with LogCapture() as log: @@ -726,7 +726,7 @@ class TestEngineThrottling: engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} engine._slot = Mock() - # A throttling-aware scheduler (one with next_request_delay) holds + # A throttling-aware scheduler (one with get_next_request_delay) holds # throttled requests itself, so no switch is recommended. engine._slot.scheduler = Mock() with LogCapture() as log: diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 5fc4207e4..c45687a3f 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -364,7 +364,7 @@ class TestThrottlingAwarePriorityQueue: # The blocked second slow request stays in the queue. assert None in urls assert len(queue) == 1 - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) @@ -385,7 +385,7 @@ class TestThrottlingAwarePriorityQueue: assert "http://fast.com/1" in urls assert None in urls assert len(queue) == 1 - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) @@ -407,7 +407,7 @@ class TestThrottlingAwarePriorityQueue: # The delayed request is not lost, just not poppable yet. assert queue.pop() is None assert len(queue) == 1 - assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0) + assert queue.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) @coroutine_test async def test_delayed_request_promoted_when_due(self): @@ -508,7 +508,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler(Spider) queue = self._queue(crawler) assert queue.pop() is None - assert queue.next_request_delay() is None + assert queue.get_next_request_delay() is None await self._push(queue, crawler, Request("http://a.com/1")) assert queue.close() != {} @@ -525,23 +525,23 @@ class TestThrottlingAwarePriorityQueue: assert len(queue) == 1 @coroutine_test - async def test_next_request_delay_zero_when_ready(self): + async def test_get_next_request_delay_zero_when_ready(self): crawler = get_crawler(Spider) queue = self._queue(crawler) await self._push(queue, crawler, Request("http://a.com/1")) # A sendable head means no wait is needed. - assert queue.next_request_delay() == 0.0 + assert queue.get_next_request_delay() == 0.0 @coroutine_test - async def test_next_request_delay_ignores_empty_queues(self): + async def test_get_next_request_delay_ignores_empty_queues(self): crawler = get_crawler(Spider) queue = self._queue(crawler) # An empty (but still registered) internal queue is skipped. queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"})) - assert queue.next_request_delay() is None + assert queue.get_next_request_delay() is None @coroutine_test - async def test_next_request_delay_keeps_minimum(self): + async def test_get_next_request_delay_keeps_minimum(self): crawler = get_crawler( Spider, settings_dict={ @@ -563,7 +563,7 @@ class TestThrottlingAwarePriorityQueue: queue.pop() # Both scopes are now time-blocked; the smaller per-scope delay wins, # so the larger one exercises the "not below the running minimum" branch. - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay == pytest.approx(10.0, abs=1.0) @coroutine_test diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 12dc61461..d1d56bbcb 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -498,7 +498,7 @@ class TestThrottlingAwareScheduler: assert first is not None assert scheduler.next_request() is None assert scheduler.has_pending_requests() - assert scheduler.next_request_delay() == pytest.approx(1000.0, abs=1.0) + assert scheduler.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) scheduler.close("finished") @coroutine_test @@ -512,7 +512,7 @@ class TestThrottlingAwareScheduler: assert scheduler.next_request() is not None assert scheduler.next_request() is None # A purely concurrency-blocked state has no time-based wakeup. - assert scheduler.next_request_delay() is None + assert scheduler.get_next_request_delay() is None scheduler.close("finished") @coroutine_test diff --git a/tests/test_throttling.py b/tests/test_throttling.py index d088d6c7e..abcb29604 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -762,7 +762,7 @@ class TestThrottlingManagerReadiness: manager.reserve(first) # The base delay now blocks any further request for the scope. assert manager.is_ready(second) is False - assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0) + assert manager.get_time_until_ready(second) == pytest.approx(100.0, abs=1.0) @coroutine_test async def test_throttling_delay_blocks_until_deadline(self): @@ -772,7 +772,7 @@ class TestThrottlingManagerReadiness: # The per-request delay holds back the request even though its scope is # otherwise unconstrained. assert manager.is_ready(request) is False - assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(100.0, abs=1.0) # The deadline is computed once and reused by later polls. deadline = request.meta["_throttling_delay_deadline"] assert manager.is_ready(request) is False @@ -813,7 +813,7 @@ class TestThrottlingManagerReadiness: # response header does. manager.delay_scope("example.com", 50.0) assert manager.is_ready(request) is False - assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(50.0, abs=1.0) @coroutine_test async def test_delay_scope_bypasses_max_delay(self): @@ -825,7 +825,7 @@ class TestThrottlingManagerReadiness: request = Request("http://example.com/a") await manager.get_scopes(request) manager.delay_scope("example.com", 1000.0) - assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(1000.0, abs=1.0) @coroutine_test async def test_reserve_blocks_on_concurrency(self): @@ -837,7 +837,7 @@ class TestThrottlingManagerReadiness: manager.reserve(first) assert manager.is_ready(second) is False # Pure concurrency blocking is not time-gated. - assert manager.time_until_ready(second) is None + assert manager.get_time_until_ready(second) is None manager.release(first) assert manager.is_ready(second) is True @@ -859,20 +859,20 @@ class TestThrottlingManagerReadiness: assert scope._active == 1 # reserve recorded exactly one send @coroutine_test - async def test_scope_load(self): + async def test_get_scope_load(self): manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 4}}}) - assert manager.scope_load("example.com") == 0.0 + assert manager.get_scope_load("example.com") == 0.0 request = Request("http://example.com/1") await manager.get_scopes(request) manager.reserve(request) - assert manager.scope_load("example.com") == pytest.approx(0.25) + assert manager.get_scope_load("example.com") == pytest.approx(0.25) - def test_scope_load_falls_back_to_global_concurrency(self): + def test_get_scope_load_falls_back_to_global_concurrency(self): manager = _manager({"CONCURRENT_REQUESTS": 8}) # A scope with no explicit concurrency limit uses CONCURRENT_REQUESTS. request = Request("http://example.com/1") manager.reserve(request) - assert manager.scope_load("example.com") == pytest.approx(1 / 8) + assert manager.get_scope_load("example.com") == pytest.approx(1 / 8) class TestParseRateHeaders: @@ -971,11 +971,11 @@ class TestThrottlingManagerEdges: await manager.acquire(request) assert request not in manager._reserved - def test_scope_load_without_concurrency_limit(self): + def test_get_scope_load_without_concurrency_limit(self): manager = _manager({"CONCURRENT_REQUESTS": 0}) # CONCURRENT_REQUESTS is 0, so the load denominator is 0 and the load is # reported as 0 instead of raising. - assert manager.scope_load("example.com") == 0.0 + assert manager.get_scope_load("example.com") == 0.0 @coroutine_test async def test_acquire_logs_and_waits_for_delay(self):