RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use priority queues for Downloader slot queues 

closes #1731
This commit is contained in:
Daniel Graña 2015-08-18 23:53:09 -03:00
parent 9adb5c31c0
commit 4e2885c9bf
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/__init__.py
View File

@ -7,6 +7,8 @@ from collections import deque
import six import six
from twisted.internet import reactor, defer, task from twisted.internet import reactor, defer, task
from queuelib.pqueue import PriorityQueue
from queuelib.queue import FifoMemoryQueue
from scrapy.utils.defer import mustbe_deferred from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
@ -25,7 +27,7 @@ class Slot(object):
self.randomize_delay = randomize_delay self.randomize_delay = randomize_delay
self.active = set() self.active = set()
self.queue = deque() self.queue = PriorityQueue(qfactory=lambda p: FifoMemoryQueue())
self.transferring = set() self.transferring = set()
self.lastseen = 0 self.lastseen = 0
self.latercall = None self.latercall = None
@ -130,7 +132,7 @@ class Downloader(object):
slot.active.add(request) slot.active.add(request)
deferred = defer.Deferred().addBoth(_deactivate) deferred = defer.Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred)) slot.queue.push((request, deferred), priority=-request.priority)
self._process_queue(spider, slot) self._process_queue(spider, slot)
return deferred return deferred
@ -150,7 +152,7 @@ class Downloader(object):
# Process enqueued requests if there are free slots to transfer for this slot # Process enqueued requests if there are free slots to transfer for this slot
while slot.queue and slot.free_transfer_slots() > 0: while slot.queue and slot.free_transfer_slots() > 0:
slot.lastseen = now slot.lastseen = now
request, deferred = slot.queue.popleft() request, deferred = slot.queue.pop()
dfd = self._download(slot, request, spider) dfd = self._download(slot, request, spider)
dfd.chainDeferred(deferred) dfd.chainDeferred(deferred)
# prevent burst if inter-request delays were configured # prevent burst if inter-request delays were configured