From 9f1742bf8d7a17c3a6e6127ed440057976b937a9 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 4 Aug 2026 18:55:42 +0200 Subject: [PATCH 1/3] Support SQLite-backed queues --- docs/topics/security.rst | 28 +++++++++++ docs/topics/settings.rst | 13 ++++- scrapy/squeues.py | 28 +++++++++++ tests/test_squeues.py | 36 +++++++++++--- tests/test_squeues_request.py | 92 ++++++++++++++++++++++++++++++++++- 5 files changed, 189 insertions(+), 8 deletions(-) diff --git a/docs/topics/security.rst b/docs/topics/security.rst index 2ca270045..22c124d67 100644 --- a/docs/topics/security.rst +++ b/docs/topics/security.rst @@ -205,3 +205,31 @@ and a crawl that spans multiple domains can leak them to unintended hosts: default :setting:`REFERRER_POLICY` already avoids sending the referrer from HTTPS to HTTP, but you can tighten it further (for example, to ``same-origin`` or ``no-referrer``) if needed. + +.. _security-job-state: + +Job state integrity +=================== + +When you use :setting:`JOBDIR` to :ref:`pause and resume crawls +`, the default scheduler disk queues +(:setting:`SCHEDULER_DISK_QUEUE`, :setting:`SCHEDULER_START_DISK_QUEUE`) write +requests to a compact binary format that is only guaranteed to be complete +after a clean shutdown. If the process is killed, runs out of disk space, or +shares its job directory with a second process, those files can be left +truncated, and resuming the crawl then either fails with an unrecoverable +error or silently drops the pending requests. + +To store the scheduler queues in an SQLite database instead, where each +request is written within its own transaction: + +.. code-block:: python + + SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoSQLiteQueue" + SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoSQLiteQueue" + +* **Pro:** an interrupted write cannot corrupt the queues or discard pending + requests unnoticed. + +* **Con:** scheduling requests becomes slower, which can be noticeable in + crawls with a high request throughput. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65ee77258..044aee429 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1885,10 +1885,21 @@ SCHEDULER_DISK_QUEUE Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` +.. versionadded:: VERSION + The ``SQLite`` queue types. + Type of disk queue that will be used by the scheduler. Other available types are ``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, -``scrapy.squeues.MarshalLifoDiskQueue``. +``scrapy.squeues.MarshalLifoDiskQueue``, +``scrapy.squeues.PickleFifoSQLiteQueue``, +``scrapy.squeues.PickleLifoSQLiteQueue``, +``scrapy.squeues.MarshalFifoSQLiteQueue`` and +``scrapy.squeues.MarshalLifoSQLiteQueue``. + +The ``SQLite`` types store requests in an SQLite database, which makes writes +slower but keeps the queue usable after an unclean shutdown. See +:ref:`security-job-state`. .. setting:: SCHEDULER_MEMORY_QUEUE diff --git a/scrapy/squeues.py b/scrapy/squeues.py index e06e38e23..ed3287d1f 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -166,11 +166,39 @@ _MarshalLifoSerializationDiskQueue = _serializable_queue( marshal.dumps, marshal.loads, ) +_PickleFifoSerializationSQLiteQueue = _serializable_queue( + _with_mkdir(queue.FifoSQLiteQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, +) +_PickleLifoSerializationSQLiteQueue = _serializable_queue( + _with_mkdir(queue.LifoSQLiteQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, +) +_MarshalFifoSerializationSQLiteQueue = _serializable_queue( + _with_mkdir(queue.FifoSQLiteQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, +) +_MarshalLifoSerializationSQLiteQueue = _serializable_queue( + _with_mkdir(queue.LifoSQLiteQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, +) # public queue classes PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) +PickleFifoSQLiteQueue = _scrapy_serialization_queue(_PickleFifoSerializationSQLiteQueue) +PickleLifoSQLiteQueue = _scrapy_serialization_queue(_PickleLifoSerializationSQLiteQueue) +MarshalFifoSQLiteQueue = _scrapy_serialization_queue( + _MarshalFifoSerializationSQLiteQueue +) +MarshalLifoSQLiteQueue = _scrapy_serialization_queue( + _MarshalLifoSerializationSQLiteQueue +) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8544602af..8f67abdfa 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -10,9 +10,13 @@ from scrapy.loader import ItemLoader from scrapy.selector import Selector from scrapy.squeues import ( _MarshalFifoSerializationDiskQueue, + _MarshalFifoSerializationSQLiteQueue, _MarshalLifoSerializationDiskQueue, + _MarshalLifoSerializationSQLiteQueue, _PickleFifoSerializationDiskQueue, + _PickleFifoSerializationSQLiteQueue, _PickleLifoSerializationDiskQueue, + _PickleLifoSerializationSQLiteQueue, ) @@ -45,7 +49,7 @@ def nonserializable_object_test(self): q.close() -class FifoDiskQueueTestMixin: +class FifoQueueTestMixin: def test_serialize(self): q = self.queue() q.push("a") @@ -59,7 +63,7 @@ class FifoDiskQueueTestMixin: test_nonserializable_object = nonserializable_object_test -class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): +class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin): chunksize = 100000 def queue(self): @@ -82,7 +86,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 -class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): +class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin): chunksize = 100000 def queue(self): @@ -154,7 +158,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 -class LifoDiskQueueTestMixin: +class LifoQueueTestMixin: def test_serialize(self): q = self.queue() q.push("a") @@ -168,12 +172,12 @@ class LifoDiskQueueTestMixin: test_nonserializable_object = nonserializable_object_test -class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): +class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin): def queue(self): return _MarshalLifoSerializationDiskQueue(self.qpath) -class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): +class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin): def queue(self): return _PickleLifoSerializationDiskQueue(self.qpath) @@ -206,3 +210,23 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): assert r.url == r2.url assert r2.meta["request"] is r2 q.close() + + +class MarshalFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin): + def queue(self): + return _MarshalFifoSerializationSQLiteQueue(self.qpath) + + +class PickleFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin): + def queue(self): + return _PickleFifoSerializationSQLiteQueue(self.qpath) + + +class MarshalLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin): + def queue(self): + return _MarshalLifoSerializationSQLiteQueue(self.qpath) + + +class PickleLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin): + def queue(self): + return _PickleLifoSerializationSQLiteQueue(self.qpath) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index c779b005f..b26299663 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -5,7 +5,7 @@ Queues that handle requests from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest import queuelib @@ -16,13 +16,19 @@ from scrapy.squeues import ( FifoMemoryQueue, LifoMemoryQueue, MarshalFifoDiskQueue, + MarshalFifoSQLiteQueue, MarshalLifoDiskQueue, + MarshalLifoSQLiteQueue, PickleFifoDiskQueue, + PickleFifoSQLiteQueue, PickleLifoDiskQueue, + PickleLifoSQLiteQueue, ) from scrapy.utils.test import get_crawler if TYPE_CHECKING: + from pathlib import Path + from scrapy.crawler import Crawler @@ -165,6 +171,90 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase): queue.close() +class TestPickleFifoSQLiteQueueRequest(TestRequestQueueBase): + is_fifo = True + + @pytest.fixture + def q(self, crawler, tmp_path): + queue = PickleFifoSQLiteQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "pickle" / "fifo") + ) + try: + yield queue + finally: + queue.close() + + +class TestPickleLifoSQLiteQueueRequest(TestRequestQueueBase): + is_fifo = False + + @pytest.fixture + def q(self, crawler, tmp_path): + queue = PickleLifoSQLiteQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "pickle" / "lifo") + ) + try: + yield queue + finally: + queue.close() + + +class TestMarshalFifoSQLiteQueueRequest(TestRequestQueueBase): + is_fifo = True + + @pytest.fixture + def q(self, crawler, tmp_path): + queue = MarshalFifoSQLiteQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "marshal" / "fifo") + ) + try: + yield queue + finally: + queue.close() + + +class TestMarshalLifoSQLiteQueueRequest(TestRequestQueueBase): + is_fifo = False + + @pytest.fixture + def q(self, crawler, tmp_path): + queue = MarshalLifoSQLiteQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "marshal" / "lifo") + ) + try: + yield queue + finally: + queue.close() + + +@pytest.mark.parametrize( + "queue_cls", + [ + PickleFifoSQLiteQueue, + PickleLifoSQLiteQueue, + MarshalFifoSQLiteQueue, + MarshalLifoSQLiteQueue, + ], +) +def test_sqlite_queue_survives_unclean_shutdown( + crawler: Crawler, tmp_path: Path, queue_cls: Any +) -> None: + key = str(tmp_path / "queue") + queue = queue_cls.from_crawler(crawler=crawler, key=key) + queue.push(Request("https://toscrape.com")) + # No close(), as if the process had been killed. + del queue + + queue = queue_cls.from_crawler(crawler=crawler, key=key) + try: + assert len(queue) == 1 + request = queue.pop() + assert request is not None + assert request.url == "https://toscrape.com" + finally: + queue.close() + + class TestFifoMemoryQueueRequest(TestRequestQueueBase): is_fifo = True From e174d76ed4452f039bf2f12808a190068696ea3b Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 4 Aug 2026 19:14:48 +0200 Subject: [PATCH 2/3] Fix tests for Windows --- tests/test_squeues_request.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b26299663..3765a6488 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -4,6 +4,7 @@ Queues that handle requests from __future__ import annotations +import gc from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any @@ -242,8 +243,11 @@ def test_sqlite_queue_survives_unclean_shutdown( key = str(tmp_path / "queue") queue = queue_cls.from_crawler(crawler=crawler, key=key) queue.push(Request("https://toscrape.com")) - # No close(), as if the process had been killed. + # No close(), as if the process had been killed. The SQLite connection is + # part of a reference cycle, so a garbage collection is needed to release + # the underlying file, which on Windows cannot be removed while open. del queue + gc.collect() queue = queue_cls.from_crawler(crawler=crawler, key=key) try: From 953c5d8a7b66ec7d6f67e820924c1c778b07de13 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 5 Aug 2026 03:12:12 +0200 Subject: [PATCH 3/3] Make tests more resilient on Windows --- tests/test_squeues_request.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 3765a6488..254262bb5 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -4,7 +4,6 @@ Queues that handle requests from __future__ import annotations -import gc from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any @@ -243,11 +242,10 @@ def test_sqlite_queue_survives_unclean_shutdown( key = str(tmp_path / "queue") queue = queue_cls.from_crawler(crawler=crawler, key=key) queue.push(Request("https://toscrape.com")) - # No close(), as if the process had been killed. The SQLite connection is - # part of a reference cycle, so a garbage collection is needed to release - # the underlying file, which on Windows cannot be removed while open. - del queue - gc.collect() + # No close(), as if the process had been killed. The connection is closed + # directly instead, since on Windows the file cannot be removed later while + # it is still open. + queue._db.close() queue = queue_cls.from_crawler(crawler=crawler, key=key) try: