From 685807e3c42ba770b9886d5530e45fe13e92a0ac Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 19 Mar 2009 03:11:27 +0000 Subject: [PATCH] profiling: improve list+deque+cache performance results: == With 1 priorities (/tmp/pq-13882-1-50000) == pushpops = 50000, times = 30 heapq implementation: 12.6956150532 dict+deque implementation: 5.3080239296 deque+heapq implementation: 3.11057305336 deque+defaultdict+deque implementation: 3.06583619118 list+deque implementation: 4.85028195381 list+deque+cache implementation: 4.86092495918 == With 3 priorities (/tmp/pq-13882-3-50000) == pushpops = 50000, times = 30 heapq implementation: 13.9048631191 dict+deque implementation: 6.526501894 deque+heapq implementation: 9.95749187469 deque+defaultdict+deque implementation: 4.94318699837 list+deque implementation: 5.48832702637 list+deque+cache implementation: 4.77395009995 == With 5 priorities (/tmp/pq-13882-5-50000) == pushpops = 50000, times = 30 heapq implementation: 14.1862449646 dict+deque implementation: 7.45535206795 deque+heapq implementation: 11.7175529003 deque+defaultdict+deque implementation: 5.40972518921 list+deque implementation: 5.87488412857 list+deque+cache implementation: 4.73579287529 == With 10 priorities (/tmp/pq-13882-10-50000) == pushpops = 50000, times = 30 heapq implementation: 14.2052979469 dict+deque implementation: 9.94834208488 deque+heapq implementation: 13.0460109711 deque+defaultdict+deque implementation: 5.79300785065 list+deque implementation: 6.9981739521 list+deque+cache implementation: 4.81988596916 == With 100 priorities (/tmp/pq-13882-100-50000) == pushpops = 50000, times = 30 heapq implementation: 14.9574189186 dict+deque implementation: 55.6348400116 deque+heapq implementation: 14.9515259266 deque+defaultdict+deque implementation: 10.6776599884 list+deque implementation: 25.9212520123 list+deque+cache implementation: 4.77596998215 --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40996 --- .../trunk/profiling/priorityqueue/pq_classes.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scrapy/trunk/profiling/priorityqueue/pq_classes.py b/scrapy/trunk/profiling/priorityqueue/pq_classes.py index aebf38e18..af0586e42 100644 --- a/scrapy/trunk/profiling/priorityqueue/pq_classes.py +++ b/scrapy/trunk/profiling/priorityqueue/pq_classes.py @@ -216,20 +216,19 @@ class PriorityQueue6(object): self.priolist = [deque() for _ in range(size)] def push(self, item, priority=0): - final = priority + self.zero - if self.index > final: - self.index = final - self.priolist[final].appendleft(item) + i = priority + self.zero + self.priolist[i].appendleft(item) + self.index = min(self.index, i) def pop(self): cached = self.priolist[self.index] - if len(cached): + if cached: return (cached.pop(), self.index - self.zero) for prio, queue in enumerate(self.priolist[self.index:]): - if len(queue): - final = prio + self.index - self.zero - return (queue.pop(), final) + if queue: + self.index += prio + return (queue.pop(), self.index - self.zero) raise IndexError("pop from an empty queue")