RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve request and scope delay support

This commit is contained in:
Adrian Chaves 2026-06-26 19:00:06 +02:00
parent ee8c5f7aa4
commit 5aae8334a5
6 changed files with 401 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
docs/topics/throttling.rst
View File

@ -290,7 +290,6 @@ minimum delays (capped at :setting:`BACKOFF_MAX_DELAY`).
.. seealso:: :setting:`REDIRECT_MAX_DELAY`
.. _crawl-delay:
robots.txt
@ -313,6 +312,47 @@ If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it
will be respected, but a warning will be logged about the discrepancy with
``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning.
.. _delay-scope:
Delaying a scope programmatically
=================================
You can delay a :ref:`throttling scope <throttling-scopes>` on demand through
:meth:`crawler.throttler.delay_scope()
<scrapy.throttling.ThrottlingManagerProtocol.delay_scope>`:
.. code-block:: python
crawler.throttler.delay_scope("example.com", 30.0)
This holds back every request of the scope for at least the given number of
seconds, counted as a :ref:`backoff <backoff>` trigger.
It is useful to react to situations that :ref:`automatic backoff <backoff>`
cannot detect on its own, such as a soft block that comes back as a ``200``
response. For example, a spider callback can slow down the whole domain when it
detects a maintenance page, and reschedule the current request:
.. code-block:: python
from scrapy import Request, Spider
from scrapy.utils.httpobj import urlparse_cached
class MySpider(Spider):
name = "myspider"
start_urls = ["https://example.com/"]
def parse(self, response):
if "under maintenance" in response.text:
scope = urlparse_cached(response).netloc
self.crawler.throttler.delay_scope(scope, 600.0)
yield response.request.replace(dont_filter=True)
return
# Normal parsing follows.
Unlike :ref:`untrusted delays <rate-limiting-headers>`, this delay is **not**
capped at :setting:`BACKOFF_MAX_DELAY`.
.. _per-request-throttling:
@ -374,6 +414,19 @@ regardless of its scopes, set the ``throttling_delay`` request metadata key:
The delay is applied once, the first time the request reaches the throttling
gate.
``throttling_delay`` defines only the *earliest* time the request may be sent,
not the exact time: once the delay elapses, the request still competes with
every other pending request for its scopes. If you want it sent **as soon as**
its delay elapses, give it a higher :attr:`~scrapy.Request.priority` too:
.. code-block:: python
Request("https://example.com/slow", meta={"throttling_delay": 5.0}, priority=1)
Without a higher priority, a backlog of requests ahead of it in a FIFO queue
could keep it waiting well past the configured delay; a higher priority puts it
at the front of the queue, so it goes out right after its delay.
.. reqmeta:: throttling_dont_track
Excluding a request from throttling state

71
scrapy/pqueues.py
View File

@ -1,8 +1,10 @@
from __future__ import annotations
import hashlib
import heapq
import json
import logging
import time
from typing import TYPE_CHECKING, Protocol, cast
from scrapy.utils.misc import build_from_crawler
@ -455,9 +457,9 @@ def _scope_set_from_key(key: str) -> frozenset[ScopeID]:
class ThrottlingAwarePriorityQueue:
"""Priority queue that partitions requests by their full :ref:`throttling
scope set <throttling-scopes>` and only ever pops a request that can be
sent right now.
"""Priority queue that only ever pops a request that can be sent right now
based on its :ref:`throttling scope set <throttling-scopes>` and
per-request :reqmeta:`throttling_delay`.
The downstream queue class must support ``peek``.
@ -520,6 +522,11 @@ class ThrottlingAwarePriorityQueue:
# scope set -> priority queue
self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {}
# Min-heap of (deadline, seq, scope_set, request) for requests held back
# by a per-request throttling_delay; seq keeps ordering stable and
# avoids comparing requests when deadlines tie.
self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = []
self._delayed_seq: int = 0
if slot_startprios:
for set_key, startprios in slot_startprios.items():
scope_set = _scope_set_from_key(set_key)
@ -537,10 +544,52 @@ class ThrottlingAwarePriorityQueue:
)
def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None:
now = time.monotonic()
self._promote_ready(now)
delay = self._throttler.get_request_delay(request, now)
if delay > 0:
self._delayed_seq += 1
heapq.heappush(
self._delayed, (now + delay, self._delayed_seq, scope_set, request)
)
return
self._push_to_queue(request, scope_set)
def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None:
if scope_set not in self.pqueues:
self.pqueues[scope_set] = self.pqfactory(scope_set)
self.pqueues[scope_set].push(request)
def _promote_ready(self, now: float) -> None:
"""Move every held-back request whose per-request delay has elapsed into
its scope-set queue, where it competes normally for its scopes."""
while self._delayed and self._delayed[0][0] <= now:
self._release_delayed(heapq.heappop(self._delayed))
def _release_delayed(
self, entry: tuple[float, int, frozenset[ScopeID], Request]
) -> None:
_, _, scope_set, request = entry
# The per-request delay has been honored (or the queue is closing), so
# mark it consumed: the request must not be delayed again, and on resume
# it must not re-block its scope set on a stale, no-longer-meaningful
# deadline.
request.meta["_throttling_delayed"] = True
try:
self._push_to_queue(request, scope_set)
except ValueError as e:
# A disk queue serializes on push; held-back requests defer that
# serialization until here, so a non-serializable one would
# otherwise raise while flushing on close and take the rest of the
# disk queue down with it. Drop it with a warning instead, matching
# how the scheduler handles unserializable requests at enqueue time.
logger.warning(
"Unable to serialize request: %(request)s - reason: %(reason)s",
{"request": request, "reason": e},
exc_info=True,
extra={"spider": getattr(self.crawler, "spider", None)},
)
def _select(
self,
) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None:
@ -553,6 +602,7 @@ class ThrottlingAwarePriorityQueue:
:meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the
scopes of the queue), i.e. by preferring the least-busy scopes.
"""
self._promote_ready(time.monotonic())
best_sort_key: tuple[int, float] | None = None
best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None
for scope_set, queue in self.pqueues.items():
@ -588,6 +638,8 @@ class ThrottlingAwarePriorityQueue:
return selected[1].peek()
def next_request_delay(self) -> float | None:
now = time.monotonic()
self._promote_ready(now)
delay: float | None = None
for queue in self.pqueues.values():
head = queue.peek()
@ -600,9 +652,19 @@ class ThrottlingAwarePriorityQueue:
continue
if delay is None or head_delay < delay:
delay = head_delay
# A request held back only by its own throttling_delay is not in any
# scope-set queue, so factor in when the earliest one is due.
if self._delayed:
next_delayed = max(0.0, self._delayed[0][0] - now)
if delay is None or next_delayed < delay:
delay = next_delayed
return delay
def close(self) -> dict[str, list[int]]:
# Flush held-back requests into their scope-set queues so they are
# persisted (and restored on resume) rather than lost.
while self._delayed:
self._release_delayed(heapq.heappop(self._delayed))
active = {
_scope_set_key(scope_set): queue.close()
for scope_set, queue in self.pqueues.items()
@ -611,7 +673,8 @@ class ThrottlingAwarePriorityQueue:
return active
def __len__(self) -> int:
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
return queued + len(self._delayed)
def __contains__(self, scope_set: frozenset[ScopeID]) -> bool:
return scope_set in self.pqueues

98
scrapy/throttling.py
View File

@ -323,10 +323,10 @@ class ThrottlingManagerProtocol(Protocol):
*and* a concurrency slot is free in every scope.
This is the synchronous, non-blocking counterpart of :meth:`acquire`,
used by a :ref:`throttling-aware scheduler <throttling-aware-scheduler>`
to decide whether a request can be dequeued now. It assumes the scopes
of *request* have already been resolved (e.g. by an earlier
:meth:`get_scopes` call at enqueue time).
used by a :ref:`throttling-aware scheduler
<throttling-aware-scheduler>` to decide whether a request can be
dequeued now. It assumes the scopes of *request* have already been
resolved (e.g. by an earlier :meth:`get_scopes` call at enqueue time).
"""
def reserve(self, request: Request) -> None:
@ -359,6 +359,31 @@ class ThrottlingManagerProtocol(Protocol):
preferring the least-loaded ones.
"""
def get_request_delay(self, request: Request, now: float | None = None) -> float:
"""Return how many seconds *request* must still be held individually
because of its :reqmeta:`throttling_delay`, or ``0.0`` if it has none
or it has already elapsed. The one-time delay is started on the first
call.
Unlike a scope delay, this affects only *request*: a
:ref:`throttling-aware scheduler <throttling-aware-scheduler>` must
hold the request back on its own, **without** blocking other requests
that share its scopes.
"""
def delay_scope(self, scope_id: str, delay: float) -> None:
"""Hold back every request of the scope identified by *scope_id* for at
least *delay* seconds, counted as a :ref:`backoff <backoff>` trigger
for the scope.
This is the programmatic equivalent of a :ref:`Retry-After
<retry-after>` response header, available to any component through
:attr:`crawler.throttler <scrapy.crawler.Crawler.throttler>`. Unlike
those headers, *delay* is **not** capped at
:setting:`BACKOFF_MAX_DELAY`: that cap guards against untrusted input,
whereas a ``delay_scope`` call is trusted.
"""
async def process_response(self, response: Response) -> None:
"""Update the throttling state based on *response*."""
@ -551,7 +576,7 @@ class ThrottlingManager:
return
now = time.monotonic()
self._maybe_evict(now)
await self._apply_request_delay(request)
await self._delay_request(request)
scope_values = list(iter_scope_values(await self.get_scopes(request)))
if not scope_values:
return
@ -599,6 +624,8 @@ class ThrottlingManager:
def is_ready(self, request: Request) -> bool:
now = time.monotonic()
if self._request_delay_deadline(request, now) > now:
return False
for scope_id, value in self._cached_scope_values(request):
manager = self._get_scope_manager(scope_id)
if manager.can_send(now=now, amount=value) > 0:
@ -618,7 +645,7 @@ class ThrottlingManager:
def time_until_ready(self, request: Request) -> float | None:
now = time.monotonic()
wait = 0.0
wait = max(0.0, self._request_delay_deadline(request, now) - now)
for scope_id, value in self._cached_scope_values(request):
manager = self._get_scope_manager(scope_id)
wait = max(wait, manager.can_send(now=now, amount=value))
@ -633,6 +660,10 @@ class ThrottlingManager:
return 0.0
return active / limit
def get_request_delay(self, request: Request, now: float | None = None) -> float:
now = time.monotonic() if now is None else now
return max(0.0, self._request_delay_deadline(request, now) - now)
async def _wait_for_slot(self, managers: list[Any]) -> None:
"""Block until any of *managers* frees a concurrency slot.
@ -649,7 +680,7 @@ class ThrottlingManager:
if event in pending:
manager.discard_slot_event(event)
async def _apply_request_delay(self, request: Request) -> None:
async def _delay_request(self, request: Request) -> None:
"""Honor the :reqmeta:`throttling_delay` meta key by holding *request*
for the requested number of seconds the first time it is processed."""
delay = request.meta.get("throttling_delay")
@ -660,6 +691,31 @@ class ThrottlingManager:
logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)")
await sleep(float(delay))
def _request_delay_deadline(self, request: Request, now: float) -> float:
"""Return the monotonic time before which *request* must not be sent due
to its :reqmeta:`throttling_delay`, or ``0.0`` if it has none.
This is the readiness-API counterpart of :meth:`_delay_request`:
a throttling-aware scheduler gates requests through :meth:`is_ready` and
:meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the
delay is enforced by holding back the request until this deadline rather
than by sleeping. The deadline is computed once, the first time the
request reaches the gate, and stored so later polls reuse it.
A request whose delay has already been honored (the ``_throttling_delayed``
flag, also set by :meth:`_delay_request`) is never delayed again,
which keeps a resumed crawl from re-blocking on a stale deadline."""
delay = request.meta.get("throttling_delay")
if not delay or request.meta.get("_throttling_delayed"):
return 0.0
deadline = request.meta.get("_throttling_delay_deadline")
if deadline is None:
deadline = now + float(delay)
request.meta["_throttling_delay_deadline"] = deadline
if self._debug:
logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)")
return deadline
async def process_response(self, response: Response) -> None:
data = await self.get_response_backoff(response)
self._apply_backoff(data)
@ -752,6 +808,13 @@ class ThrottlingManager:
manager.set_base_delay(capped)
manager.set_concurrency(1)
def delay_scope(self, scope_id: ScopeID, delay: float) -> None:
if self._debug:
logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s")
# Like a Retry-After / RateLimit-Reset header, this is a hard minimum
# delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY.
self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False)
def _maybe_evict(self, now: float) -> None:
if self._max_idle <= 0:
return
@ -823,13 +886,21 @@ class ThrottlingScopeManagerProtocol(Protocol):
downloading, freeing its concurrency slot."""
def record_backoff(
self, delay: float | None = None, now: float | None = None
self,
delay: float | None = None,
now: float | None = None,
cap: bool = True,
) -> None:
"""Apply a backoff to this scope.
*delay*, when given, is a hard minimum delay in seconds (e.g. from a
``Retry-After`` header). When omitted, an exponential backoff step is
applied instead.
*cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`. It is ``True`` for
untrusted input such as response headers, and may be set to ``False``
for trusted, programmatic delays (see
:meth:`ThrottlingManagerProtocol.delay_scope`).
"""
def reconcile_quota(
@ -1128,7 +1199,10 @@ class ThrottlingScopeManager:
event.callback(None)
def record_backoff(
self, delay: float | None = None, now: float | None = None
self,
delay: float | None = None,
now: float | None = None,
cap: bool = True,
) -> None:
now = self._now(now)
self._last_seen = now
@ -1136,9 +1210,11 @@ class ThrottlingScopeManager:
self._backoff_level += 1
self._rampup_backoffs += 1
if delay is not None:
hard = min(float(delay), self._max_delay)
hard = min(float(delay), self._max_delay) if cap else float(delay)
self._in_backoff_until = now + hard
self._delay = min(max(self._delay, hard, self._min_delay), self._max_delay)
self._delay = max(self._delay, hard, self._min_delay)
if cap:
self._delay = min(self._delay, self._max_delay)
else:
grown = (
self._delay * self._delay_factor if self._delay > 0 else self._min_delay

92
tests/test_pqueues.py
View File

@ -368,6 +368,98 @@ class TestThrottlingAwarePriorityQueue:
assert delay is not None
assert delay == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_pop_holds_request_with_throttling_delay(self):
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
queue = self._queue(crawler)
await self._push(
queue,
crawler,
Request("http://slow.com/1", meta={"throttling_delay": 1000.0}),
)
await self._push(queue, crawler, Request("http://fast.com/1"))
# The delayed request is held back even though its scope is otherwise
# unconstrained; the request without a delay is served.
popped = [queue.pop(), queue.pop()]
urls = [r.url if r else None for r in popped]
assert "http://fast.com/1" in urls
assert None in urls
assert len(queue) == 1
delay = queue.next_request_delay()
assert delay is not None
assert delay == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_delayed_request_does_not_block_scope_set(self):
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
queue = self._queue(crawler)
# Both requests share the same (example.com) scope set; only the first
# carries a per-request delay.
await self._push(
queue,
crawler,
Request("http://example.com/slow", meta={"throttling_delay": 1000.0}),
)
await self._push(queue, crawler, Request("http://example.com/fast"))
# The delayed request is held aside, so the other request in the same
# scope set is served right away instead of being stuck behind it.
assert queue.pop().url == "http://example.com/fast"
# The delayed request is not lost, just not poppable yet.
assert queue.pop() is None
assert len(queue) == 1
assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_delayed_request_promoted_when_due(self):
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
queue = self._queue(crawler)
request = Request("http://example.com/slow", meta={"throttling_delay": 1000.0})
await self._push(queue, crawler, request)
assert queue.pop() is None # held back by its per-request delay
# Once the delay elapses the request is promoted into its scope-set
# queue, served, and flagged so the delay is not applied a second time.
queue._promote_ready(queue._delayed[0][0])
popped = queue.pop()
assert popped is request
assert popped.meta["_throttling_delayed"] is True
assert len(queue) == 0
@coroutine_test
async def test_delayed_request_persisted_on_close(self):
# With a JOBDIR (disk queue), a request held back by its per-request
# delay must not be lost on a graceful stop: close() flushes it to disk
# so it is restored on resume.
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
temp_dir = tempfile.mkdtemp()
queue = build_from_crawler(
ThrottlingAwarePriorityQueue,
crawler,
downstream_queue_cls=PickleFifoDiskQueue,
key=temp_dir,
)
await self._push(
queue,
crawler,
Request("http://example.com/slow", meta={"throttling_delay": 1000.0}),
)
assert len(queue) == 1 # held in memory, not yet in any scope-set queue
state = queue.close() # graceful stop
resumed = build_from_crawler(
ThrottlingAwarePriorityQueue,
crawler,
downstream_queue_cls=PickleFifoDiskQueue,
key=temp_dir,
startprios=state,
)
assert len(resumed) == 1
popped = resumed.pop()
assert popped is not None
assert popped.url == "http://example.com/slow"
# Its delay is marked consumed, so it does not re-block on resume.
assert popped.meta["_throttling_delayed"] is True
resumed.close()
@coroutine_test
async def test_least_loaded_first(self):
crawler = get_crawler(

27
tests/test_scheduler.py
View File

@ -515,6 +515,33 @@ class TestThrottlingAwareScheduler:
assert scheduler.next_request_delay() is None
scheduler.close("finished")
@coroutine_test
async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None:
# A request held back by its per-request throttling_delay must not be
# lost on a graceful stop when a JOBDIR is configured: it is flushed to
# the disk queue on close and restored on resume.
crawler = self._crawler(
{"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False}
)
scheduler = self._scheduler(crawler)
request = Request("http://a.com/slow", meta={"throttling_delay": 1000.0})
assert await scheduler.enqueue_request_async(request) is True
assert len(scheduler) == 1
# The delay holds it back, so nothing is dequeued before the stop.
assert scheduler.next_request() is None
scheduler.close("finished")
# Resume from the same JOBDIR: the request is still there and, having
# been held once, is now sendable.
resumed = self._scheduler(
self._crawler({"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False})
)
assert len(resumed) == 1
resumed_request = resumed.next_request()
assert resumed_request is not None
assert resumed_request.url == "http://a.com/slow"
resumed.close("finished")
@coroutine_test
async def test_enqueue_async_filters_duplicates(self) -> None:
crawler = self._crawler(

79
tests/test_throttling.py
View File

@ -381,6 +381,12 @@ class TestThrottlingScopeManager:
scope.record_backoff(delay=backoff_delay, now=0.0)
assert scope.can_send(now=0.0) == pytest.approx(expected)
def test_uncapped_backoff_delay(self):
# cap=False (used by trusted delay_scope() calls) ignores BACKOFF_MAX_DELAY.
scope = _scope_manager({"BACKOFF_MAX_DELAY": 10.0})
scope.record_backoff(delay=999.0, now=0.0, cap=False)
assert scope.can_send(now=0.0) == pytest.approx(999.0)
def test_recovery_after_window(self):
scope = _scope_manager(
{
@ -598,6 +604,69 @@ class TestThrottlingManagerReadiness:
assert manager.is_ready(second) is False
assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0)
@coroutine_test
async def test_throttling_delay_blocks_until_deadline(self):
manager = _manager({"THROTTLING_DEBUG": True})
request = Request("http://example.com/a", meta={"throttling_delay": 100.0})
await manager.get_scopes(request)
# The per-request delay holds back the request even though its scope is
# otherwise unconstrained.
assert manager.is_ready(request) is False
assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0)
# The deadline is computed once and reused by later polls.
deadline = request.meta["_throttling_delay_deadline"]
assert manager.is_ready(request) is False
assert request.meta["_throttling_delay_deadline"] == deadline
@coroutine_test
async def test_throttling_delay_not_reapplied_once_consumed(self):
# A request whose delay was already honored (e.g. promoted out of a
# throttling-aware queue's holding area, or restored on resume) is
# ready, so it cannot re-block its scope set on a stale deadline.
manager = _manager()
request = Request(
"http://example.com/a",
meta={"throttling_delay": 100.0, "_throttling_delayed": True},
)
await manager.get_scopes(request)
assert manager.is_ready(request) is True
assert manager.get_request_delay(request) == 0.0
@coroutine_test
async def test_get_request_delay(self):
manager = _manager()
assert manager.get_request_delay(
Request("http://example.com/a", meta={"throttling_delay": 100.0})
) == pytest.approx(100.0, abs=1.0)
# A request without a per-request delay is not held individually.
assert manager.get_request_delay(Request("http://example.com/b")) == 0.0
@coroutine_test
async def test_delay_scope(self):
manager = _manager(
{"THROTTLING_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False}
)
request = Request("http://example.com/a")
await manager.get_scopes(request)
assert manager.is_ready(request) is True
# A component can delay a whole scope on demand, like a Retry-After
# response header does.
manager.delay_scope("example.com", 50.0)
assert manager.is_ready(request) is False
assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0)
@coroutine_test
async def test_delay_scope_bypasses_max_delay(self):
# BACKOFF_MAX_DELAY caps untrusted input (headers), but delay_scope is a
# trusted call, so it may exceed the cap.
manager = _manager(
{"BACKOFF_MAX_DELAY": 30.0, "RANDOMIZE_DOWNLOAD_DELAY": False}
)
request = Request("http://example.com/a")
await manager.get_scopes(request)
manager.delay_scope("example.com", 1000.0)
assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0)
@coroutine_test
async def test_reserve_blocks_on_concurrency(self):
manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}})
@ -783,21 +852,21 @@ class TestThrottlingManagerEdges:
assert r2 in manager._reserved
@coroutine_test
async def test_apply_request_delay(self):
async def test_delay_request(self):
manager = _manager({"THROTTLING_DEBUG": True})
request = Request("http://example.com/a", meta={"throttling_delay": 0.01})
await manager._apply_request_delay(request)
await manager._delay_request(request)
assert request.meta["_throttling_delayed"] is True
# A second call is a no-op (the request was already delayed).
await manager._apply_request_delay(request)
await manager._delay_request(request)
@coroutine_test
async def test_apply_request_delay_without_debug(self):
async def test_delay_request_without_debug(self):
# Same as above but with debug logging off, so the delay is applied
# without emitting the debug message.
manager = _manager()
request = Request("http://example.com/a", meta={"throttling_delay": 0.01})
await manager._apply_request_delay(request)
await manager._delay_request(request)
assert request.meta["_throttling_delayed"] is True
@coroutine_test