diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d807fe347..82e31b90b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -131,10 +131,10 @@ class Scheduler(BaseScheduler): (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by :attr:`~scrapy.http.Request.priority`. - By default, a single, memory-based priority queue is used for all requests. - When using :setting:`JOBDIR`, a disk-based priority queue is also created, + By default, memory-based priority queues are used for all requests. + When using :setting:`JOBDIR`, disk-based priority queues are also created, and only unserializable requests are stored in the memory-based priority - queue. For a given priority value, requests in memory take precedence over + queues. For a given priority value, requests in memory take precedence over requests in disk. Each priority queue stores requests in separate internal queues, one per @@ -342,7 +342,7 @@ class Scheduler(BaseScheduler): def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue - (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (2) initialize the disk queue if the ``jobdir`` argument wasn't empty (3) return the result of the dupefilter's ``open`` method """ self.spider: Spider = spider diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f125cae4..ac5651dc7 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error. You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry - -Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 2efc43d4b..41411ceaf 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -92,9 +92,10 @@ class ScrapyPriorityQueue: - The :data:`~scrapy.Request.priority` of the request. For each combination of the above seen, this class creates an instance of - *downstream_queue_cls* with *key* set to a subdirectory of the persistence - directory, named as the request priority (e.g. ``1``), with an ``s`` suffix - in case of a start request (e.g. ``1s``). + *downstream_queue_cls* (or *start_queue_cls* for start requests if it was + passed) with *key* set to a subdirectory of the persistence directory, + named as the negated request priority (e.g. ``-1``), with an ``s`` suffix + in case of a start request (e.g. ``-1s``). """ @classmethod