diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 57d27b7cf..9ac447289 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -173,8 +173,8 @@ class Scheduler(BaseScheduler): :ref:`Start requests ` are sent in the order they are yielded from :meth:`~scrapy.Spider.start`, and given the same - :attr:`~scrapy.http.Request.priority`, start requests take precedence over - other requests. + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e6c6b8bf1..34b235d83 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -160,28 +160,32 @@ class ScrapyPriorityQueue: def pop(self) -> Request | None: while self.curprio is not None: - if self._start_queues: - try: - q = self._start_queues[self.curprio] - except KeyError: - pass - else: - m = q.pop() - if not q: - del self._start_queues[self.curprio] - q.close() - return m try: q = self.queues[self.curprio] except KeyError: - self._update_curprio() + pass else: m = q.pop() if not q: del self.queues[self.curprio] q.close() - self._update_curprio() + if not self._start_queues: + self._update_curprio() return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() return None def _update_curprio(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 90af10f0e..c7dbc82d4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase): @deferred_f_from_coro_f async def test_default(self): - """By default, start requests take priority over callback requests and + """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a - start request takes precedence.""" + callback request takes precedence.""" nums = [1, 2, 3, 4, 5, 6] response_seconds = 0 download_slots = 1 @@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) - yield _request(2, priority=1) + yield _request(6) yield _request(3, priority=1) + yield _request(4, priority=1) def parse(spider, response): return @@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) + yield _request(6) + yield _request(4, priority=1) yield _request(3, priority=1) - yield _request(2, priority=1) def parse(spider, response): return