From 4e2885c9bfbd43ce009b7113eb12f342135ea3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 18 Aug 2015 23:53:09 -0300 Subject: [PATCH] Use priority queues for Downloader slot queues closes #1731 --- scrapy/core/downloader/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 8beb08159..1ad6ff35f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -7,6 +7,8 @@ from collections import deque import six from twisted.internet import reactor, defer, task +from queuelib.pqueue import PriorityQueue +from queuelib.queue import FifoMemoryQueue from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached @@ -25,7 +27,7 @@ class Slot(object): self.randomize_delay = randomize_delay self.active = set() - self.queue = deque() + self.queue = PriorityQueue(qfactory=lambda p: FifoMemoryQueue()) self.transferring = set() self.lastseen = 0 self.latercall = None @@ -130,7 +132,7 @@ class Downloader(object): slot.active.add(request) deferred = defer.Deferred().addBoth(_deactivate) - slot.queue.append((request, deferred)) + slot.queue.push((request, deferred), priority=-request.priority) self._process_queue(spider, slot) return deferred @@ -150,7 +152,7 @@ class Downloader(object): # Process enqueued requests if there are free slots to transfer for this slot while slot.queue and slot.free_transfer_slots() > 0: slot.lastseen = now - request, deferred = slot.queue.popleft() + request, deferred = slot.queue.pop() dfd = self._download(slot, request, spider) dfd.chainDeferred(deferred) # prevent burst if inter-request delays were configured