diff --git a/scrapy/trunk/profiling/priorityqueue/pq_classes.py b/scrapy/trunk/profiling/priorityqueue/pq_classes.py index af0586e42..f9d22c74e 100644 --- a/scrapy/trunk/profiling/priorityqueue/pq_classes.py +++ b/scrapy/trunk/profiling/priorityqueue/pq_classes.py @@ -245,4 +245,21 @@ class PriorityQueue6(object): return bool(len(self)) +from itertools import islice +class PriorityQueue6b(PriorityQueue6): + """Priority queue using a deque for priority 0""" + def pop(self): + cached = self.priolist[self.index] + if cached: + return (cached.pop(), self.index - self.zero) + + for prio, queue in enumerate(islice(self.priolist, self.index, None), self.index): + if queue: + self.index = prio + return (queue.pop(), prio - self.zero) + + raise IndexError("pop from an empty queue") + + +__all__ = [name for name in globals().keys() if name.startswith('PriorityQueue')] diff --git a/scrapy/trunk/profiling/priorityqueue/run.py b/scrapy/trunk/profiling/priorityqueue/run.py index 91f02e25f..fe9885b81 100644 --- a/scrapy/trunk/profiling/priorityqueue/run.py +++ b/scrapy/trunk/profiling/priorityqueue/run.py @@ -15,6 +15,7 @@ TESTCASES = [ ("deque+defaultdict+deque", PriorityQueue4), ('list+deque', PriorityQueue5), ('list+deque+cache', PriorityQueue6), + ('list+deque+cache+islice', PriorityQueue6b), ]