mirror of https://github.com/scrapy/scrapy.git
Queue stuff.
This commit is contained in:
parent
8573026049
commit
e543c54b2c
|
|
@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):
|
|||
(:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
|
||||
:attr:`~scrapy.http.Request.priority`.
|
||||
|
||||
By default, a single, memory-based priority queue is used for all requests.
|
||||
When using :setting:`JOBDIR`, a disk-based priority queue is also created,
|
||||
By default, memory-based priority queues are used for all requests.
|
||||
When using :setting:`JOBDIR`, disk-based priority queues are also created,
|
||||
and only unserializable requests are stored in the memory-based priority
|
||||
queue. For a given priority value, requests in memory take precedence over
|
||||
queues. For a given priority value, requests in memory take precedence over
|
||||
requests in disk.
|
||||
|
||||
Each priority queue stores requests in separate internal queues, one per
|
||||
|
|
@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):
|
|||
def open(self, spider: Spider) -> Deferred[None] | None:
|
||||
"""
|
||||
(1) initialize the memory queue
|
||||
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
|
||||
(2) initialize the disk queue if the ``jobdir`` argument wasn't empty
|
||||
(3) return the result of the dupefilter's ``open`` method
|
||||
"""
|
||||
self.spider: Spider = spider
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.
|
|||
You can change the behaviour of this middleware by modifying the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non-failed) pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -92,9 +92,10 @@ class ScrapyPriorityQueue:
|
|||
- The :data:`~scrapy.Request.priority` of the request.
|
||||
|
||||
For each combination of the above seen, this class creates an instance of
|
||||
*downstream_queue_cls* with *key* set to a subdirectory of the persistence
|
||||
directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``1s``).
|
||||
*downstream_queue_cls* (or *start_queue_cls* for start requests if it was
|
||||
passed) with *key* set to a subdirectory of the persistence directory,
|
||||
named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``-1s``).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
Loading…
Reference in New Issue