diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 332a874c5..8d3ae004e 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -290,7 +290,6 @@ minimum delays (capped at :setting:`BACKOFF_MAX_DELAY`). .. seealso:: :setting:`REDIRECT_MAX_DELAY` - .. _crawl-delay: robots.txt @@ -313,6 +312,47 @@ If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it will be respected, but a warning will be logged about the discrepancy with ``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. +.. _delay-scope: + +Delaying a scope programmatically +================================= + +You can delay a :ref:`throttling scope ` on demand through +:meth:`crawler.throttler.delay_scope() +`: + +.. code-block:: python + + crawler.throttler.delay_scope("example.com", 30.0) + +This holds back every request of the scope for at least the given number of +seconds, counted as a :ref:`backoff ` trigger. + +It is useful to react to situations that :ref:`automatic backoff ` +cannot detect on its own, such as a soft block that comes back as a ``200`` +response. For example, a spider callback can slow down the whole domain when it +detects a maintenance page, and reschedule the current request: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.utils.httpobj import urlparse_cached + + + class MySpider(Spider): + name = "myspider" + start_urls = ["https://example.com/"] + + def parse(self, response): + if "under maintenance" in response.text: + scope = urlparse_cached(response).netloc + self.crawler.throttler.delay_scope(scope, 600.0) + yield response.request.replace(dont_filter=True) + return + # Normal parsing follows. + +Unlike :ref:`untrusted delays `, this delay is **not** +capped at :setting:`BACKOFF_MAX_DELAY`. .. _per-request-throttling: @@ -374,6 +414,19 @@ regardless of its scopes, set the ``throttling_delay`` request metadata key: The delay is applied once, the first time the request reaches the throttling gate. +``throttling_delay`` defines only the *earliest* time the request may be sent, +not the exact time: once the delay elapses, the request still competes with +every other pending request for its scopes. If you want it sent **as soon as** +its delay elapses, give it a higher :attr:`~scrapy.Request.priority` too: + +.. code-block:: python + + Request("https://example.com/slow", meta={"throttling_delay": 5.0}, priority=1) + +Without a higher priority, a backlog of requests ahead of it in a FIFO queue +could keep it waiting well past the configured delay; a higher priority puts it +at the front of the queue, so it goes out right after its delay. + .. reqmeta:: throttling_dont_track Excluding a request from throttling state diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 60675fe7f..75dc757db 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,8 +1,10 @@ from __future__ import annotations import hashlib +import heapq import json import logging +import time from typing import TYPE_CHECKING, Protocol, cast from scrapy.utils.misc import build_from_crawler @@ -455,9 +457,9 @@ def _scope_set_from_key(key: str) -> frozenset[ScopeID]: class ThrottlingAwarePriorityQueue: - """Priority queue that partitions requests by their full :ref:`throttling - scope set ` and only ever pops a request that can be - sent right now. + """Priority queue that only ever pops a request that can be sent right now + based on its :ref:`throttling scope set ` and + per-request :reqmeta:`throttling_delay`. The downstream queue class must support ``peek``. @@ -520,6 +522,11 @@ class ThrottlingAwarePriorityQueue: # scope set -> priority queue self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} + # Min-heap of (deadline, seq, scope_set, request) for requests held back + # by a per-request throttling_delay; seq keeps ordering stable and + # avoids comparing requests when deadlines tie. + self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] + self._delayed_seq: int = 0 if slot_startprios: for set_key, startprios in slot_startprios.items(): scope_set = _scope_set_from_key(set_key) @@ -537,10 +544,52 @@ class ThrottlingAwarePriorityQueue: ) def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None: + now = time.monotonic() + self._promote_ready(now) + delay = self._throttler.get_request_delay(request, now) + if delay > 0: + self._delayed_seq += 1 + heapq.heappush( + self._delayed, (now + delay, self._delayed_seq, scope_set, request) + ) + return + self._push_to_queue(request, scope_set) + + def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None: if scope_set not in self.pqueues: self.pqueues[scope_set] = self.pqfactory(scope_set) self.pqueues[scope_set].push(request) + def _promote_ready(self, now: float) -> None: + """Move every held-back request whose per-request delay has elapsed into + its scope-set queue, where it competes normally for its scopes.""" + while self._delayed and self._delayed[0][0] <= now: + self._release_delayed(heapq.heappop(self._delayed)) + + def _release_delayed( + self, entry: tuple[float, int, frozenset[ScopeID], Request] + ) -> None: + _, _, scope_set, request = entry + # The per-request delay has been honored (or the queue is closing), so + # mark it consumed: the request must not be delayed again, and on resume + # it must not re-block its scope set on a stale, no-longer-meaningful + # deadline. + request.meta["_throttling_delayed"] = True + try: + self._push_to_queue(request, scope_set) + except ValueError as e: + # A disk queue serializes on push; held-back requests defer that + # serialization until here, so a non-serializable one would + # otherwise raise while flushing on close and take the rest of the + # disk queue down with it. Drop it with a warning instead, matching + # how the scheduler handles unserializable requests at enqueue time. + logger.warning( + "Unable to serialize request: %(request)s - reason: %(reason)s", + {"request": request, "reason": e}, + exc_info=True, + extra={"spider": getattr(self.crawler, "spider", None)}, + ) + def _select( self, ) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None: @@ -553,6 +602,7 @@ class ThrottlingAwarePriorityQueue: :meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the scopes of the queue), i.e. by preferring the least-busy scopes. """ + self._promote_ready(time.monotonic()) best_sort_key: tuple[int, float] | None = None best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None for scope_set, queue in self.pqueues.items(): @@ -588,6 +638,8 @@ class ThrottlingAwarePriorityQueue: return selected[1].peek() def next_request_delay(self) -> float | None: + now = time.monotonic() + self._promote_ready(now) delay: float | None = None for queue in self.pqueues.values(): head = queue.peek() @@ -600,9 +652,19 @@ class ThrottlingAwarePriorityQueue: continue if delay is None or head_delay < delay: delay = head_delay + # A request held back only by its own throttling_delay is not in any + # scope-set queue, so factor in when the earliest one is due. + if self._delayed: + next_delayed = max(0.0, self._delayed[0][0] - now) + if delay is None or next_delayed < delay: + delay = next_delayed return delay def close(self) -> dict[str, list[int]]: + # Flush held-back requests into their scope-set queues so they are + # persisted (and restored on resume) rather than lost. + while self._delayed: + self._release_delayed(heapq.heappop(self._delayed)) active = { _scope_set_key(scope_set): queue.close() for scope_set, queue in self.pqueues.items() @@ -611,7 +673,8 @@ class ThrottlingAwarePriorityQueue: return active def __len__(self) -> int: - return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + return queued + len(self._delayed) def __contains__(self, scope_set: frozenset[ScopeID]) -> bool: return scope_set in self.pqueues diff --git a/scrapy/throttling.py b/scrapy/throttling.py index f90a25dd8..e603f1fbe 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -323,10 +323,10 @@ class ThrottlingManagerProtocol(Protocol): *and* a concurrency slot is free in every scope. This is the synchronous, non-blocking counterpart of :meth:`acquire`, - used by a :ref:`throttling-aware scheduler ` - to decide whether a request can be dequeued now. It assumes the scopes - of *request* have already been resolved (e.g. by an earlier - :meth:`get_scopes` call at enqueue time). + used by a :ref:`throttling-aware scheduler + ` to decide whether a request can be + dequeued now. It assumes the scopes of *request* have already been + resolved (e.g. by an earlier :meth:`get_scopes` call at enqueue time). """ def reserve(self, request: Request) -> None: @@ -359,6 +359,31 @@ class ThrottlingManagerProtocol(Protocol): preferring the least-loaded ones. """ + def get_request_delay(self, request: Request, now: float | None = None) -> float: + """Return how many seconds *request* must still be held individually + because of its :reqmeta:`throttling_delay`, or ``0.0`` if it has none + or it has already elapsed. The one-time delay is started on the first + call. + + Unlike a scope delay, this affects only *request*: a + :ref:`throttling-aware scheduler ` must + hold the request back on its own, **without** blocking other requests + that share its scopes. + """ + + def delay_scope(self, scope_id: str, delay: float) -> None: + """Hold back every request of the scope identified by *scope_id* for at + least *delay* seconds, counted as a :ref:`backoff ` trigger + for the scope. + + This is the programmatic equivalent of a :ref:`Retry-After + ` response header, available to any component through + :attr:`crawler.throttler `. Unlike + those headers, *delay* is **not** capped at + :setting:`BACKOFF_MAX_DELAY`: that cap guards against untrusted input, + whereas a ``delay_scope`` call is trusted. + """ + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -551,7 +576,7 @@ class ThrottlingManager: return now = time.monotonic() self._maybe_evict(now) - await self._apply_request_delay(request) + await self._delay_request(request) scope_values = list(iter_scope_values(await self.get_scopes(request))) if not scope_values: return @@ -599,6 +624,8 @@ class ThrottlingManager: def is_ready(self, request: Request) -> bool: now = time.monotonic() + if self._request_delay_deadline(request, now) > now: + return False for scope_id, value in self._cached_scope_values(request): manager = self._get_scope_manager(scope_id) if manager.can_send(now=now, amount=value) > 0: @@ -618,7 +645,7 @@ class ThrottlingManager: def time_until_ready(self, request: Request) -> float | None: now = time.monotonic() - wait = 0.0 + wait = max(0.0, self._request_delay_deadline(request, now) - now) for scope_id, value in self._cached_scope_values(request): manager = self._get_scope_manager(scope_id) wait = max(wait, manager.can_send(now=now, amount=value)) @@ -633,6 +660,10 @@ class ThrottlingManager: return 0.0 return active / limit + def get_request_delay(self, request: Request, now: float | None = None) -> float: + now = time.monotonic() if now is None else now + return max(0.0, self._request_delay_deadline(request, now) - now) + async def _wait_for_slot(self, managers: list[Any]) -> None: """Block until any of *managers* frees a concurrency slot. @@ -649,7 +680,7 @@ class ThrottlingManager: if event in pending: manager.discard_slot_event(event) - async def _apply_request_delay(self, request: Request) -> None: + async def _delay_request(self, request: Request) -> None: """Honor the :reqmeta:`throttling_delay` meta key by holding *request* for the requested number of seconds the first time it is processed.""" delay = request.meta.get("throttling_delay") @@ -660,6 +691,31 @@ class ThrottlingManager: logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") await sleep(float(delay)) + def _request_delay_deadline(self, request: Request, now: float) -> float: + """Return the monotonic time before which *request* must not be sent due + to its :reqmeta:`throttling_delay`, or ``0.0`` if it has none. + + This is the readiness-API counterpart of :meth:`_delay_request`: + a throttling-aware scheduler gates requests through :meth:`is_ready` and + :meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the + delay is enforced by holding back the request until this deadline rather + than by sleeping. The deadline is computed once, the first time the + request reaches the gate, and stored so later polls reuse it. + + A request whose delay has already been honored (the ``_throttling_delayed`` + flag, also set by :meth:`_delay_request`) is never delayed again, + which keeps a resumed crawl from re-blocking on a stale deadline.""" + delay = request.meta.get("throttling_delay") + if not delay or request.meta.get("_throttling_delayed"): + return 0.0 + deadline = request.meta.get("_throttling_delay_deadline") + if deadline is None: + deadline = now + float(delay) + request.meta["_throttling_delay_deadline"] = deadline + if self._debug: + logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") + return deadline + async def process_response(self, response: Response) -> None: data = await self.get_response_backoff(response) self._apply_backoff(data) @@ -752,6 +808,13 @@ class ThrottlingManager: manager.set_base_delay(capped) manager.set_concurrency(1) + def delay_scope(self, scope_id: ScopeID, delay: float) -> None: + if self._debug: + logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s") + # Like a Retry-After / RateLimit-Reset header, this is a hard minimum + # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. + self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + def _maybe_evict(self, now: float) -> None: if self._max_idle <= 0: return @@ -823,13 +886,21 @@ class ThrottlingScopeManagerProtocol(Protocol): downloading, freeing its concurrency slot.""" def record_backoff( - self, delay: float | None = None, now: float | None = None + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, ) -> None: """Apply a backoff to this scope. *delay*, when given, is a hard minimum delay in seconds (e.g. from a ``Retry-After`` header). When omitted, an exponential backoff step is applied instead. + + *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`. It is ``True`` for + untrusted input such as response headers, and may be set to ``False`` + for trusted, programmatic delays (see + :meth:`ThrottlingManagerProtocol.delay_scope`). """ def reconcile_quota( @@ -1128,7 +1199,10 @@ class ThrottlingScopeManager: event.callback(None) def record_backoff( - self, delay: float | None = None, now: float | None = None + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, ) -> None: now = self._now(now) self._last_seen = now @@ -1136,9 +1210,11 @@ class ThrottlingScopeManager: self._backoff_level += 1 self._rampup_backoffs += 1 if delay is not None: - hard = min(float(delay), self._max_delay) + hard = min(float(delay), self._max_delay) if cap else float(delay) self._in_backoff_until = now + hard - self._delay = min(max(self._delay, hard, self._min_delay), self._max_delay) + self._delay = max(self._delay, hard, self._min_delay) + if cap: + self._delay = min(self._delay, self._max_delay) else: grown = ( self._delay * self._delay_factor if self._delay > 0 else self._min_delay diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 92dfcafe5..5fc4207e4 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -368,6 +368,98 @@ class TestThrottlingAwarePriorityQueue: assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) + @coroutine_test + async def test_pop_holds_request_with_throttling_delay(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + await self._push( + queue, + crawler, + Request("http://slow.com/1", meta={"throttling_delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://fast.com/1")) + # The delayed request is held back even though its scope is otherwise + # unconstrained; the request without a delay is served. + popped = [queue.pop(), queue.pop()] + urls = [r.url if r else None for r in popped] + assert "http://fast.com/1" in urls + assert None in urls + assert len(queue) == 1 + delay = queue.next_request_delay() + assert delay is not None + assert delay == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_does_not_block_scope_set(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + # Both requests share the same (example.com) scope set; only the first + # carries a per-request delay. + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://example.com/fast")) + # The delayed request is held aside, so the other request in the same + # scope set is served right away instead of being stuck behind it. + assert queue.pop().url == "http://example.com/fast" + # The delayed request is not lost, just not poppable yet. + assert queue.pop() is None + assert len(queue) == 1 + assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_promoted_when_due(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + request = Request("http://example.com/slow", meta={"throttling_delay": 1000.0}) + await self._push(queue, crawler, request) + assert queue.pop() is None # held back by its per-request delay + # Once the delay elapses the request is promoted into its scope-set + # queue, served, and flagged so the delay is not applied a second time. + queue._promote_ready(queue._delayed[0][0]) + popped = queue.pop() + assert popped is request + assert popped.meta["_throttling_delayed"] is True + assert len(queue) == 0 + + @coroutine_test + async def test_delayed_request_persisted_on_close(self): + # With a JOBDIR (disk queue), a request held back by its per-request + # delay must not be lost on a graceful stop: close() flushes it to disk + # so it is restored on resume. + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + temp_dir = tempfile.mkdtemp() + queue = build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + ) + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + ) + assert len(queue) == 1 # held in memory, not yet in any scope-set queue + state = queue.close() # graceful stop + + resumed = build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + startprios=state, + ) + assert len(resumed) == 1 + popped = resumed.pop() + assert popped is not None + assert popped.url == "http://example.com/slow" + # Its delay is marked consumed, so it does not re-block on resume. + assert popped.meta["_throttling_delayed"] is True + resumed.close() + @coroutine_test async def test_least_loaded_first(self): crawler = get_crawler( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ad91efcc7..12dc61461 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -515,6 +515,33 @@ class TestThrottlingAwareScheduler: assert scheduler.next_request_delay() is None scheduler.close("finished") + @coroutine_test + async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None: + # A request held back by its per-request throttling_delay must not be + # lost on a graceful stop when a JOBDIR is configured: it is flushed to + # the disk queue on close and restored on resume. + crawler = self._crawler( + {"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + scheduler = self._scheduler(crawler) + request = Request("http://a.com/slow", meta={"throttling_delay": 1000.0}) + assert await scheduler.enqueue_request_async(request) is True + assert len(scheduler) == 1 + # The delay holds it back, so nothing is dequeued before the stop. + assert scheduler.next_request() is None + scheduler.close("finished") + + # Resume from the same JOBDIR: the request is still there and, having + # been held once, is now sendable. + resumed = self._scheduler( + self._crawler({"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False}) + ) + assert len(resumed) == 1 + resumed_request = resumed.next_request() + assert resumed_request is not None + assert resumed_request.url == "http://a.com/slow" + resumed.close("finished") + @coroutine_test async def test_enqueue_async_filters_duplicates(self) -> None: crawler = self._crawler( diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 62efe29bb..3dc7727f9 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -381,6 +381,12 @@ class TestThrottlingScopeManager: scope.record_backoff(delay=backoff_delay, now=0.0) assert scope.can_send(now=0.0) == pytest.approx(expected) + def test_uncapped_backoff_delay(self): + # cap=False (used by trusted delay_scope() calls) ignores BACKOFF_MAX_DELAY. + scope = _scope_manager({"BACKOFF_MAX_DELAY": 10.0}) + scope.record_backoff(delay=999.0, now=0.0, cap=False) + assert scope.can_send(now=0.0) == pytest.approx(999.0) + def test_recovery_after_window(self): scope = _scope_manager( { @@ -598,6 +604,69 @@ class TestThrottlingManagerReadiness: assert manager.is_ready(second) is False assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0) + @coroutine_test + async def test_throttling_delay_blocks_until_deadline(self): + manager = _manager({"THROTTLING_DEBUG": True}) + request = Request("http://example.com/a", meta={"throttling_delay": 100.0}) + await manager.get_scopes(request) + # The per-request delay holds back the request even though its scope is + # otherwise unconstrained. + assert manager.is_ready(request) is False + assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0) + # The deadline is computed once and reused by later polls. + deadline = request.meta["_throttling_delay_deadline"] + assert manager.is_ready(request) is False + assert request.meta["_throttling_delay_deadline"] == deadline + + @coroutine_test + async def test_throttling_delay_not_reapplied_once_consumed(self): + # A request whose delay was already honored (e.g. promoted out of a + # throttling-aware queue's holding area, or restored on resume) is + # ready, so it cannot re-block its scope set on a stale deadline. + manager = _manager() + request = Request( + "http://example.com/a", + meta={"throttling_delay": 100.0, "_throttling_delayed": True}, + ) + await manager.get_scopes(request) + assert manager.is_ready(request) is True + assert manager.get_request_delay(request) == 0.0 + + @coroutine_test + async def test_get_request_delay(self): + manager = _manager() + assert manager.get_request_delay( + Request("http://example.com/a", meta={"throttling_delay": 100.0}) + ) == pytest.approx(100.0, abs=1.0) + # A request without a per-request delay is not held individually. + assert manager.get_request_delay(Request("http://example.com/b")) == 0.0 + + @coroutine_test + async def test_delay_scope(self): + manager = _manager( + {"THROTTLING_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + assert manager.is_ready(request) is True + # A component can delay a whole scope on demand, like a Retry-After + # response header does. + manager.delay_scope("example.com", 50.0) + assert manager.is_ready(request) is False + assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0) + + @coroutine_test + async def test_delay_scope_bypasses_max_delay(self): + # BACKOFF_MAX_DELAY caps untrusted input (headers), but delay_scope is a + # trusted call, so it may exceed the cap. + manager = _manager( + {"BACKOFF_MAX_DELAY": 30.0, "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + manager.delay_scope("example.com", 1000.0) + assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0) + @coroutine_test async def test_reserve_blocks_on_concurrency(self): manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) @@ -783,21 +852,21 @@ class TestThrottlingManagerEdges: assert r2 in manager._reserved @coroutine_test - async def test_apply_request_delay(self): + async def test_delay_request(self): manager = _manager({"THROTTLING_DEBUG": True}) request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) - await manager._apply_request_delay(request) + await manager._delay_request(request) assert request.meta["_throttling_delayed"] is True # A second call is a no-op (the request was already delayed). - await manager._apply_request_delay(request) + await manager._delay_request(request) @coroutine_test - async def test_apply_request_delay_without_debug(self): + async def test_delay_request_without_debug(self): # Same as above but with debug logging off, so the delay is applied # without emitting the debug message. manager = _manager() request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) - await manager._apply_request_delay(request) + await manager._delay_request(request) assert request.meta["_throttling_delayed"] is True @coroutine_test