From 6c1852d313d7aae81e28c4146676aa10ab85251d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Apr 2025 11:07:20 +0200 Subject: [PATCH] Update the FAQ --- docs/conf.py | 18 ++++++++++++++++++ docs/faq.rst | 23 +++++++++-------------- scrapy/squeues.py | 22 +++++++++++++++++++++- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1447b253f..34407d817 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -177,3 +177,21 @@ hoverxref_role_types = { hoverxref_roles = ["command", "reqmeta", "setting", "signal"] default_dark_mode = False + +# -- setup ------------------------------------------------------------------- + + +def setup(app): + # https://stackoverflow.com/a/58982001 + for cls_name in ( + "FifoMemoryQueue", + "LifoMemoryQueue", + "PickleFifoDiskQueue", + "PickleLifoDiskQueue", + "MarshalFifoDiskQueue", + "MarshalLifoDiskQueue", + ): + import scrapy.squeues + + cls = getattr(scrapy.squeues, cls_name) + cls.__name__ = cls_name diff --git a/docs/faq.rst b/docs/faq.rst index da255f29e..3d01ebe25 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -105,27 +105,22 @@ By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which basically means that it crawls in `DFO order`_. This order is more convenient in most cases. -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: +To crawl in `BFO order`_: -.. code-block:: python +- Set :setting:`DEPTH_PRIORITY` to ``1``. - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" +- Set :setting:`SCHEDULER_MEMORY_QUEUE` to + :class:`~scrapy.squeues.FifoMemoryQueue`. -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +- Set :setting:`SCHEDULER_DISK_QUEUE` to + :class:`~scrapy.squeues.PickleFifoDiskQueue`. +- :ref:`Send start requests before other requests `. My Scrapy crawler has memory leaks. What can I do? -------------------------------------------------- -See :ref:`topics-leaks`. +See :ref:`optimize-memory` and :ref:`topics-leaks`. Also, Python has a builtin memory leak issue which is described in :ref:`topics-leaks-without-leaks`. @@ -133,7 +128,7 @@ Also, Python has a builtin memory leak issue which is described in How can I make Scrapy consume less memory? ------------------------------------------ -See previous question. +See the previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7007cd4b8..6d73d66f0 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -167,10 +167,30 @@ _MarshalLifoSerializationDiskQueue = _serializable_queue( marshal.loads, ) -# public queue classes +# Public queue classes + +#: FIFO_ disk queue that serializes :ref:`requests ` using +#: :mod:`pickle`. +#: +#: .. _FIFO: https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics) PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) + +#: LIFO_ disk queue that serializes :ref:`requests ` using +#: :mod:`pickle`. +#: +#: .. _LIFO: https://en.wikipedia.org/wiki/LIFO_(computing) PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) + +#: FIFO_ disk queue that serializes :ref:`requests ` using +#: :mod:`marshal`. MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) + +#: LIFO_ disk queue that serializes :ref:`requests ` using +#: :mod:`marshal`. MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) + +#: FIFO_ memory queue. FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] + +#: LIFO_ memory queue. LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type]