Minimum queuelib: 1.4.2 → 1.6.1

docs/topics/throttling.rst

@@ -331,6 +331,10 @@ apply different throttling based on request priority.
 Use the ``throttling_scopes`` request metadata to assign requests to custom
 throttling groups:
 
+.. invisible-code-block: python
+
+    from scrapy.http import Request
+
 .. code-block:: python
 
     Request("https://api.example/", meta={"throttling_scopes": "api"})

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "parsel>=1.5.0",
     "protego>=0.1.15",
     "pyOpenSSL>=22.0.0",
-    "queuelib>=1.4.2",
+    "queuelib>=1.6.1",
     "service_identity>=23.1.0",
     "tldextract",
     "w3lib>=1.17.0",

scrapy/core/scheduler.py

@@ -500,6 +500,22 @@ class Scheduler(BaseScheduler):
             json.dump(state, f)
 
 
+def _queue_supports_peek(queue_cls: type) -> bool:
+    """Return whether *queue_cls* (a Scrapy queue class) is backed by an
+    underlying queue that really implements ``peek``.
+
+    Scrapy's queue wrappers in :mod:`scrapy.squeues` always define a ``peek``
+    method, but it merely delegates to the underlying queue class (e.g. a
+    queuelib one, which only gained ``peek`` in queuelib 1.6.1) and raises
+    :exc:`NotImplementedError` if that one lacks it. This checks the real
+    implementation by ignoring the delegating wrappers.
+    """
+    return any(
+        "peek" in base.__dict__ and base.__module__ != "scrapy.squeues"
+        for base in queue_cls.__mro__
+    )
+
+
 class ThrottlingAwareScheduler(Scheduler):
     """A :class:`Scheduler` that only ever hands the engine requests whose
     :ref:`throttling scopes <throttling-scopes>` allow them to be sent **right
@@ -535,6 +551,19 @@ class ThrottlingAwareScheduler(Scheduler):
                 f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the "
                 f"configured one ({type(self.mqs).__name__}) is not."
             )
+        for setting, pq in (
+            ("SCHEDULER_MEMORY_QUEUE", self.mqs),
+            ("SCHEDULER_DISK_QUEUE", self.dqs),
+        ):
+            if pq is None:
+                continue
+            queue_cls = getattr(pq, "downstream_queue_cls", None)
+            if queue_cls is not None and not _queue_supports_peek(queue_cls):
+                raise ValueError(
+                    f"{type(self).__name__} requires {setting} to be set to a "
+                    f"queue class that supports peek(), but the configured one "
+                    f"({queue_cls.__name__}) does not."
+                )
         assert self.crawler is not None
         assert self.crawler.throttler is not None
         self._throttler: ThrottlingManagerProtocol = self.crawler.throttler

tests/test_scheduler.py

@@ -26,6 +26,9 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
     from pathlib import Path
 
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class MemoryScheduler(BaseScheduler):
     paused = False
@@ -411,6 +414,16 @@ class TestIncompatibility:
 _THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue"
 
 
+class _NoPeekMemoryQueue:
+    """A memory queue class that does not implement ``peek``, used to check
+    that ThrottlingAwareScheduler rejects queues lacking peek support (e.g. when
+    queuelib is older than 1.6.1)."""
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
+        return cls()
+
+
 class TestThrottlingAwareScheduler:
     def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler:
         settings = {
@@ -456,6 +469,16 @@ class TestThrottlingAwareScheduler:
         with pytest.raises(ValueError, match="throttling-aware priority queue"):
             scheduler.open(spider)
 
+    def test_requires_peek_supporting_queue(self) -> None:
+        crawler = self._crawler(
+            {"SCHEDULER_MEMORY_QUEUE": "tests.test_scheduler._NoPeekMemoryQueue"}
+        )
+        spider = Spider(name="spider")
+        crawler.spider = spider
+        scheduler = ThrottlingAwareScheduler.from_crawler(crawler)
+        with pytest.raises(ValueError, match="supports peek"):
+            scheduler.open(spider)
+
     @coroutine_test
     async def test_delay_blocks_and_reports_delay(self) -> None:
         crawler = self._crawler(

tox.ini

@@ -143,7 +143,7 @@ deps =
     lxml==4.6.4
     parsel==1.5.0
     pyOpenSSL==22.0.0
-    queuelib==1.4.2
+    queuelib==1.6.1
     service_identity==23.1.0
     w3lib==1.17.0
     zope.interface==5.1.0
@@ -259,7 +259,7 @@ deps =
     lxml==5.3.2
     parsel==1.5.0
     pyOpenSSL==24.3.0
-    queuelib==1.4.2
+    queuelib==1.6.1
     service_identity==23.1.0
     # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
     # an inline flag placement that Python 3.11 treats as an error: global