diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index cc8519990..332a874c5 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -331,6 +331,10 @@ apply different throttling based on request priority. Use the ``throttling_scopes`` request metadata to assign requests to custom throttling groups: +.. invisible-code-block: python + + from scrapy.http import Request + .. code-block:: python Request("https://api.example/", meta={"throttling_scopes": "api"}) diff --git a/pyproject.toml b/pyproject.toml index 6b0c561f8..e3d1b9989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "parsel>=1.5.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", + "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index add1891db..76eb3b5fe 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -500,6 +500,22 @@ class Scheduler(BaseScheduler): json.dump(state, f) +def _queue_supports_peek(queue_cls: type) -> bool: + """Return whether *queue_cls* (a Scrapy queue class) is backed by an + underlying queue that really implements ``peek``. + + Scrapy's queue wrappers in :mod:`scrapy.squeues` always define a ``peek`` + method, but it merely delegates to the underlying queue class (e.g. a + queuelib one, which only gained ``peek`` in queuelib 1.6.1) and raises + :exc:`NotImplementedError` if that one lacks it. This checks the real + implementation by ignoring the delegating wrappers. + """ + return any( + "peek" in base.__dict__ and base.__module__ != "scrapy.squeues" + for base in queue_cls.__mro__ + ) + + class ThrottlingAwareScheduler(Scheduler): """A :class:`Scheduler` that only ever hands the engine requests whose :ref:`throttling scopes ` allow them to be sent **right @@ -535,6 +551,19 @@ class ThrottlingAwareScheduler(Scheduler): f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the " f"configured one ({type(self.mqs).__name__}) is not." ) + for setting, pq in ( + ("SCHEDULER_MEMORY_QUEUE", self.mqs), + ("SCHEDULER_DISK_QUEUE", self.dqs), + ): + if pq is None: + continue + queue_cls = getattr(pq, "downstream_queue_cls", None) + if queue_cls is not None and not _queue_supports_peek(queue_cls): + raise ValueError( + f"{type(self).__name__} requires {setting} to be set to a " + f"queue class that supports peek(), but the configured one " + f"({queue_cls.__name__}) does not." + ) assert self.crawler is not None assert self.crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = self.crawler.throttler diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9919b05a0..94ef5b699 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -26,6 +26,9 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryScheduler(BaseScheduler): paused = False @@ -411,6 +414,16 @@ class TestIncompatibility: _THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue" +class _NoPeekMemoryQueue: + """A memory queue class that does not implement ``peek``, used to check + that ThrottlingAwareScheduler rejects queues lacking peek support (e.g. when + queuelib is older than 1.6.1).""" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: + return cls() + + class TestThrottlingAwareScheduler: def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: settings = { @@ -456,6 +469,16 @@ class TestThrottlingAwareScheduler: with pytest.raises(ValueError, match="throttling-aware priority queue"): scheduler.open(spider) + def test_requires_peek_supporting_queue(self) -> None: + crawler = self._crawler( + {"SCHEDULER_MEMORY_QUEUE": "tests.test_scheduler._NoPeekMemoryQueue"} + ) + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlingAwareScheduler.from_crawler(crawler) + with pytest.raises(ValueError, match="supports peek"): + scheduler.open(spider) + @coroutine_test async def test_delay_blocks_and_reports_delay(self) -> None: crawler = self._crawler( diff --git a/tox.ini b/tox.ini index 2fa7b455d..770067b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 w3lib==1.17.0 zope.interface==5.1.0 @@ -259,7 +259,7 @@ deps = lxml==5.3.2 parsel==1.5.0 pyOpenSSL==24.3.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses # an inline flag placement that Python 3.11 treats as an error: global