RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use get_* for getters

This commit is contained in:
Adrian Chaves 2026-06-29 10:58:18 +02:00
parent 7a211960c9
commit 184166d629
8 changed files with 49 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/core/engine.py
View File

@ -140,7 +140,7 @@ class ExecutionEngine:
self._scheduling: int = 0
# A coalesced wakeup timer, armed when a throttling-aware scheduler
# reports that all pending requests are time-blocked (see
# ``next_request_delay``).
# ``get_next_request_delay``).
self._throttling_wakeup: CallLaterResult | None = None
self._delayed_requests_warn_threshold: int = self.settings.getint(
"DELAYED_REQUESTS_WARN_THRESHOLD"
@ -376,7 +376,7 @@ class ExecutionEngine:
"""
assert self._slot is not None
scheduler = self._slot.scheduler
delay_fn = getattr(scheduler, "next_request_delay", None)
delay_fn = getattr(scheduler, "get_next_request_delay", None)
if delay_fn is None or not scheduler.has_pending_requests():
return
delay = delay_fn()
@ -433,7 +433,7 @@ class ExecutionEngine:
# scheduler instead of in _throttling_waiting, so it does not hit this
# path; recommend it only when it is not already in use.
scheduler = self._slot.scheduler if self._slot is not None else None
if scheduler is None or not hasattr(scheduler, "next_request_delay"):
if scheduler is None or not hasattr(scheduler, "get_next_request_delay"):
recommendation = (
" Consider switching to scrapy.core.scheduler.ThrottlingAwareScheduler."
)

6
scrapy/core/scheduler.py
View File

@ -544,7 +544,7 @@ class ThrottlingAwareScheduler(Scheduler):
def open(self, spider: Spider) -> Deferred[None] | None:
result = super().open(spider)
if not hasattr(self.mqs, "next_request_delay"):
if not hasattr(self.mqs, "get_next_request_delay"):
raise ValueError(
f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be "
f"set to a throttling-aware priority queue such as "
@ -619,7 +619,7 @@ class ThrottlingAwareScheduler(Scheduler):
return False
return True
def next_request_delay(self) -> float | None:
def get_next_request_delay(self) -> float | None:
"""Return the minimum number of seconds until some pending request
becomes sendable because a time-based throttling gate opens, or
``None`` if no pending request is time-blocked.
@ -630,6 +630,6 @@ class ThrottlingAwareScheduler(Scheduler):
delays = [
delay
for pq in (self.mqs, self.dqs)
if pq is not None and (delay := pq.next_request_delay()) is not None # type: ignore[attr-defined]
if pq is not None and (delay := pq.get_next_request_delay()) is not None # type: ignore[attr-defined]
]
return min(delays) if delays else None

8
scrapy/pqueues.py
View File

@ -603,7 +603,7 @@ class ThrottlingAwarePriorityQueue:
Among the sendable queues (those whose scope set can be sent right now),
the one whose head has the highest request priority is chosen; ties are
broken by ascending load (the maximum
:meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the
:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scope_load` over the
scopes of the queue), i.e. by preferring the least-busy scopes.
"""
self._promote_ready(time.monotonic())
@ -614,7 +614,7 @@ class ThrottlingAwarePriorityQueue:
if head is None or not self._throttler.is_ready(head):
continue
load = max(
(self._throttler.scope_load(scope_id) for scope_id in scope_set),
(self._throttler.get_scope_load(scope_id) for scope_id in scope_set),
default=0.0,
)
sort_key = (queue.priority(head), load)
@ -641,7 +641,7 @@ class ThrottlingAwarePriorityQueue:
return None
return selected[1].peek()
def next_request_delay(self) -> float | None:
def get_next_request_delay(self) -> float | None:
now = time.monotonic()
self._promote_ready(now)
delay: float | None = None
@ -651,7 +651,7 @@ class ThrottlingAwarePriorityQueue:
continue
if self._throttler.is_ready(head):
return 0.0
head_delay = self._throttler.time_until_ready(head)
head_delay = self._throttler.get_time_until_ready(head)
if head_delay is None:
continue
if delay is None or head_delay < delay:

20
scrapy/throttling.py
View File

@ -342,7 +342,7 @@ class ThrottlingManagerProtocol(Protocol):
returned ``True``). The reservation is released by :meth:`release`.
"""
def time_until_ready(self, request: Request) -> float | None:
def get_time_until_ready(self, request: Request) -> float | None:
"""Return the number of seconds until every time-based gate of
*request* would be open, or ``None`` if no time-based gate is currently
blocking it (only a concurrency slot could be).
@ -352,7 +352,7 @@ class ThrottlingManagerProtocol(Protocol):
requests are time-blocked.
"""
def scope_load(self, scope_id: str) -> float:
def get_scope_load(self, scope_id: str) -> float:
"""Return the current load of the scope identified by *scope_id*: its
active sends divided by its concurrency limit (or by the global
:setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit).
@ -510,7 +510,7 @@ class ThrottlingManager:
It backs :meth:`get_scopes` and is also the fallback for the synchronous
readiness methods (:meth:`is_ready`, :meth:`reserve`,
:meth:`time_until_ready`) when no scopes were persisted on
:meth:`get_time_until_ready`) when no scopes were persisted on
``request.meta`` by an earlier :meth:`get_scopes` call (which normally
happens at enqueue time and survives disk restores; see
:func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be
@ -712,7 +712,7 @@ class ThrottlingManager:
manager.record_sent(amount=value)
self._reserved[request] = managers
def time_until_ready(self, request: Request) -> float | None:
def get_time_until_ready(self, request: Request) -> float | None:
now = time.monotonic()
wait = max(0.0, self._request_delay_deadline(request, now) - now)
for scope_id, value in self._cached_scope_values(request):
@ -720,8 +720,8 @@ class ThrottlingManager:
wait = max(wait, manager.can_send(now=now, amount=value))
return wait if wait > 0 else None
def scope_load(self, scope_id: ScopeID) -> float:
return self._get_scope_manager(scope_id).load()
def get_scope_load(self, scope_id: ScopeID) -> float:
return self._get_scope_manager(scope_id).get_load()
def get_request_delay(self, request: Request, now: float | None = None) -> float:
now = time.monotonic() if now is None else now
@ -760,7 +760,7 @@ class ThrottlingManager:
This is the readiness-API counterpart of :meth:`_delay_request`:
a throttling-aware scheduler gates requests through :meth:`is_ready` and
:meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the
:meth:`get_time_until_ready` instead of awaiting :meth:`acquire`, so the
delay is enforced by holding back the request until this deadline rather
than by sleeping. The deadline is computed once, the first time the
request reaches the gate, and stored so later polls reuse it.
@ -996,7 +996,7 @@ class ThrottlingScopeManagerProtocol(Protocol):
Return ``False`` when no concurrency limit is enforced.
"""
def load(self) -> float:
def get_load(self) -> float:
"""Return the current load of this scope: a non-negative number, with
``1.0`` meaning "as busy as its concurrency limit allows".
@ -1123,7 +1123,7 @@ class ThrottlingScopeManager:
else:
self._concurrency = None
# Used as the load denominator when the scope enforces no explicit
# concurrency limit (see load()).
# concurrency limit (see get_load()).
self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS")
# Quota.
@ -1281,7 +1281,7 @@ class ThrottlingScopeManager:
def concurrency_blocked(self) -> bool:
return self._concurrency is not None and self._active >= self._concurrency
def load(self) -> float:
def get_load(self) -> float:
limit = (
self._concurrency
if self._concurrency is not None

10
tests/test_engine.py
View File

@ -677,7 +677,7 @@ class TestEngineThrottling:
def test_maybe_arm_throttling_wakeup_arms_timer(self, engine):
scheduler = Mock()
scheduler.has_pending_requests.return_value = True
scheduler.next_request_delay.return_value = 5.0
scheduler.get_next_request_delay.return_value = 5.0
engine._slot = Mock()
engine._slot.scheduler = scheduler
engine._maybe_arm_throttling_wakeup()
@ -688,7 +688,7 @@ class TestEngineThrottling:
def test_maybe_arm_throttling_wakeup_no_delay(self, engine):
scheduler = Mock()
scheduler.has_pending_requests.return_value = True
scheduler.next_request_delay.return_value = None
scheduler.get_next_request_delay.return_value = None
engine._slot = Mock()
engine._slot.scheduler = scheduler
engine._maybe_arm_throttling_wakeup()
@ -700,7 +700,7 @@ class TestEngineThrottling:
# the engine, so no timer must be armed.
scheduler = Mock()
scheduler.has_pending_requests.return_value = True
scheduler.next_request_delay.return_value = 0.0
scheduler.get_next_request_delay.return_value = 0.0
engine._slot = Mock()
engine._slot.scheduler = scheduler
engine._maybe_arm_throttling_wakeup()
@ -710,7 +710,7 @@ class TestEngineThrottling:
engine._delayed_requests_warn_threshold = 1
engine._throttling_waiting = {Request("http://a.example")}
engine._slot = Mock()
# A scheduler without next_request_delay is not throttling-aware, so the
# A scheduler without get_next_request_delay is not throttling-aware, so the
# warning recommends switching to one.
engine._slot.scheduler = Mock(spec=BaseScheduler)
with LogCapture() as log:
@ -726,7 +726,7 @@ class TestEngineThrottling:
engine._delayed_requests_warn_threshold = 1
engine._throttling_waiting = {Request("http://a.example")}
engine._slot = Mock()
# A throttling-aware scheduler (one with next_request_delay) holds
# A throttling-aware scheduler (one with get_next_request_delay) holds
# throttled requests itself, so no switch is recommended.
engine._slot.scheduler = Mock()
with LogCapture() as log:

20
tests/test_pqueues.py
View File

@ -364,7 +364,7 @@ class TestThrottlingAwarePriorityQueue:
# The blocked second slow request stays in the queue.
assert None in urls
assert len(queue) == 1
delay = queue.next_request_delay()
delay = queue.get_next_request_delay()
assert delay is not None
assert delay == pytest.approx(1000.0, abs=1.0)
@ -385,7 +385,7 @@ class TestThrottlingAwarePriorityQueue:
assert "http://fast.com/1" in urls
assert None in urls
assert len(queue) == 1
delay = queue.next_request_delay()
delay = queue.get_next_request_delay()
assert delay is not None
assert delay == pytest.approx(1000.0, abs=1.0)
@ -407,7 +407,7 @@ class TestThrottlingAwarePriorityQueue:
# The delayed request is not lost, just not poppable yet.
assert queue.pop() is None
assert len(queue) == 1
assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0)
assert queue.get_next_request_delay() == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_delayed_request_promoted_when_due(self):
@ -508,7 +508,7 @@ class TestThrottlingAwarePriorityQueue:
crawler = get_crawler(Spider)
queue = self._queue(crawler)
assert queue.pop() is None
assert queue.next_request_delay() is None
assert queue.get_next_request_delay() is None
await self._push(queue, crawler, Request("http://a.com/1"))
assert queue.close() != {}
@ -525,23 +525,23 @@ class TestThrottlingAwarePriorityQueue:
assert len(queue) == 1
@coroutine_test
async def test_next_request_delay_zero_when_ready(self):
async def test_get_next_request_delay_zero_when_ready(self):
crawler = get_crawler(Spider)
queue = self._queue(crawler)
await self._push(queue, crawler, Request("http://a.com/1"))
# A sendable head means no wait is needed.
assert queue.next_request_delay() == 0.0
assert queue.get_next_request_delay() == 0.0
@coroutine_test
async def test_next_request_delay_ignores_empty_queues(self):
async def test_get_next_request_delay_ignores_empty_queues(self):
crawler = get_crawler(Spider)
queue = self._queue(crawler)
# An empty (but still registered) internal queue is skipped.
queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"}))
assert queue.next_request_delay() is None
assert queue.get_next_request_delay() is None
@coroutine_test
async def test_next_request_delay_keeps_minimum(self):
async def test_get_next_request_delay_keeps_minimum(self):
crawler = get_crawler(
Spider,
settings_dict={
@ -563,7 +563,7 @@ class TestThrottlingAwarePriorityQueue:
queue.pop()
# Both scopes are now time-blocked; the smaller per-scope delay wins,
# so the larger one exercises the "not below the running minimum" branch.
delay = queue.next_request_delay()
delay = queue.get_next_request_delay()
assert delay == pytest.approx(10.0, abs=1.0)
@coroutine_test

4
tests/test_scheduler.py
View File

@ -498,7 +498,7 @@ class TestThrottlingAwareScheduler:
assert first is not None
assert scheduler.next_request() is None
assert scheduler.has_pending_requests()
assert scheduler.next_request_delay() == pytest.approx(1000.0, abs=1.0)
assert scheduler.get_next_request_delay() == pytest.approx(1000.0, abs=1.0)
scheduler.close("finished")
@coroutine_test
@ -512,7 +512,7 @@ class TestThrottlingAwareScheduler:
assert scheduler.next_request() is not None
assert scheduler.next_request() is None
# A purely concurrency-blocked state has no time-based wakeup.
assert scheduler.next_request_delay() is None
assert scheduler.get_next_request_delay() is None
scheduler.close("finished")
@coroutine_test

24
tests/test_throttling.py
View File

@ -762,7 +762,7 @@ class TestThrottlingManagerReadiness:
manager.reserve(first)
# The base delay now blocks any further request for the scope.
assert manager.is_ready(second) is False
assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0)
assert manager.get_time_until_ready(second) == pytest.approx(100.0, abs=1.0)
@coroutine_test
async def test_throttling_delay_blocks_until_deadline(self):
@ -772,7 +772,7 @@ class TestThrottlingManagerReadiness:
# The per-request delay holds back the request even though its scope is
# otherwise unconstrained.
assert manager.is_ready(request) is False
assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0)
assert manager.get_time_until_ready(request) == pytest.approx(100.0, abs=1.0)
# The deadline is computed once and reused by later polls.
deadline = request.meta["_throttling_delay_deadline"]
assert manager.is_ready(request) is False
@ -813,7 +813,7 @@ class TestThrottlingManagerReadiness:
# response header does.
manager.delay_scope("example.com", 50.0)
assert manager.is_ready(request) is False
assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0)
assert manager.get_time_until_ready(request) == pytest.approx(50.0, abs=1.0)
@coroutine_test
async def test_delay_scope_bypasses_max_delay(self):
@ -825,7 +825,7 @@ class TestThrottlingManagerReadiness:
request = Request("http://example.com/a")
await manager.get_scopes(request)
manager.delay_scope("example.com", 1000.0)
assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0)
assert manager.get_time_until_ready(request) == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_reserve_blocks_on_concurrency(self):
@ -837,7 +837,7 @@ class TestThrottlingManagerReadiness:
manager.reserve(first)
assert manager.is_ready(second) is False
# Pure concurrency blocking is not time-gated.
assert manager.time_until_ready(second) is None
assert manager.get_time_until_ready(second) is None
manager.release(first)
assert manager.is_ready(second) is True
@ -859,20 +859,20 @@ class TestThrottlingManagerReadiness:
assert scope._active == 1 # reserve recorded exactly one send
@coroutine_test
async def test_scope_load(self):
async def test_get_scope_load(self):
manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 4}}})
assert manager.scope_load("example.com") == 0.0
assert manager.get_scope_load("example.com") == 0.0
request = Request("http://example.com/1")
await manager.get_scopes(request)
manager.reserve(request)
assert manager.scope_load("example.com") == pytest.approx(0.25)
assert manager.get_scope_load("example.com") == pytest.approx(0.25)
def test_scope_load_falls_back_to_global_concurrency(self):
def test_get_scope_load_falls_back_to_global_concurrency(self):
manager = _manager({"CONCURRENT_REQUESTS": 8})
# A scope with no explicit concurrency limit uses CONCURRENT_REQUESTS.
request = Request("http://example.com/1")
manager.reserve(request)
assert manager.scope_load("example.com") == pytest.approx(1 / 8)
assert manager.get_scope_load("example.com") == pytest.approx(1 / 8)
class TestParseRateHeaders:
@ -971,11 +971,11 @@ class TestThrottlingManagerEdges:
await manager.acquire(request)
assert request not in manager._reserved
def test_scope_load_without_concurrency_limit(self):
def test_get_scope_load_without_concurrency_limit(self):
manager = _manager({"CONCURRENT_REQUESTS": 0})
# CONCURRENT_REQUESTS is 0, so the load denominator is 0 and the load is
# reported as 0 instead of raising.
assert manager.scope_load("example.com") == 0.0
assert manager.get_scope_load("example.com") == 0.0
@coroutine_test
async def test_acquire_logs_and_waits_for_delay(self):