From 4d1e01a4d421a425f8de21abfe6a51ba759e7dc6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 31 Jul 2011 02:47:14 -0300 Subject: [PATCH] removed obsolete profiling/ dir --- profiling/priorityqueue/pq_classes.py | 370 -------------------------- profiling/priorityqueue/run.py | 157 ----------- profiling/priorityqueue/test_cases.py | 71 ----- 3 files changed, 598 deletions(-) delete mode 100644 profiling/priorityqueue/pq_classes.py delete mode 100644 profiling/priorityqueue/run.py delete mode 100644 profiling/priorityqueue/test_cases.py diff --git a/profiling/priorityqueue/pq_classes.py b/profiling/priorityqueue/pq_classes.py deleted file mode 100644 index ff2747822..000000000 --- a/profiling/priorityqueue/pq_classes.py +++ /dev/null @@ -1,370 +0,0 @@ -from collections import deque, defaultdict -from heapq import heappush, heappop -import time -from itertools import chain - -#------------------------------------------------------------------------------ - -class PriorityQueue1(object): - """heapq - - A simple priority queue - """ - - def __init__(self, size=1): - self.items = [] - - def push(self, item, priority=0): - heappush(self.items, (priority, time.time(), item)) - - def pop(self): - priority, _, item = heappop(self.items) - return item, priority - - def __len__(self): - return len(self.items) - - def __iter__(self): - return ((item, priority) for priority, _, item in self.items) - - def __nonzero__(self): - return bool(self.items) - - -class PriorityQueue1b(PriorityQueue1): - """heapq+int - - A simple priority queue using incremental integer instead of time.time() - """ - time = 0 - - def push(self, item, priority=0): - self.time += 1 - heappush(self.items, (priority, time, item)) - -#------------------------------------------------------------------------------ - -class PriorityQueue2(object): - """dict+deque - - @author: Federico Feroldi - """ - def __init__(self, size=1): - self.items = defaultdict(deque) - - def push(self, item, priority=0): - self.items[priority].appendleft(item) - - def pop(self): - priorities = self.items.keys() - priorities.sort() - - for priority in priorities: - if len(self.items[priority]) > 0: - return (self.items[priority].pop(), priority) - - raise IndexError - - def __len__(self): - totlen = 0 - for q in self.items.values(): - totlen += len(q) - return totlen - - def __iter__(self): - priorities = self.items.keys() - priorities.sort() - - for priority in priorities: - for i in self.items[priority]: - yield (i, priority) - - def __nonzero__(self): - for q in self.items.values(): - if len(q) > 0: - return True - return False - -#------------------------------------------------------------------------------ - -class PriorityQueue3(object): - """deque+heapq""" - - def __init__(self, size=1): - self.negitems = [] - self.pzero = deque() - self.positems = [] - - def push(self, item, priority=0): - if priority == 0: - self.pzero.appendleft(item) - elif priority < 0: - heappush(self.negitems, (priority, time.time(), item)) - else: - heappush(self.positems, (priority, time.time(), item)) - - def pop(self): - if self.negitems: - priority, _, item = heappop(self.negitems) - return item, priority - elif self.pzero: - return (self.pzero.pop(), 0) - else: - priority, _, item = heappop(self.positems) - return item, priority - - def __len__(self): - return len(self.negitems) + len(self.pzero) + len(self.positems) - - def __iter__(self): - for priority, _, item in self.negitems: - yield (item, priority) - for item in self.pzero: - yield (item, 0) - for priority, _, item in self.positems: - yield (item, priority) - - def __nonzero__(self): - return bool(self.negitems and self.pzero and self.positems) - - -class PriorityQueue3b(PriorityQueue3): - """deque+heapq+int""" - - left_time = 0 - right_time = 0 - - def push(self, item, priority=0): - if priority == 0: - self.pzero.appendleft(item) - elif priority < 0: - self.left_time += 1 - heappush(self.negitems, (priority, self.left_time, item)) - else: - self.right_time += 1 - heappush(self.positems, (priority, self.right_time, item)) - - -#------------------------------------------------------------------------------ - -class PriorityQueue4(object): - """deque+defaultdict+deque""" - - def __init__(self, size=1): - self.negitems = defaultdict(deque) - self.pzero = deque() - self.positems = defaultdict(deque) - - def push(self, item, priority=0): - if priority == 0: - self.pzero.appendleft(item) - elif priority < 0: - self.negitems[priority].appendleft(item) - else: - self.positems[priority].appendleft(item) - - def pop(self): - if self.negitems: - priorities = self.negitems.keys() - priorities.sort() - for priority in priorities: - deq = self.negitems[priority] - if deq: - t = (deq.pop(), priority) - if not deq: - del self.negitems[priority] - return t - elif self.pzero: - return (self.pzero.pop(), 0) - else: - priorities = self.positems.keys() - priorities.sort() - for priority in priorities: - deq = self.positems[priority] - if deq: - t = (deq.pop(), priority) - if not deq: - del self.positems[priority] - return t - raise IndexError("pop from an empty queue") - - def __len__(self): - total = sum(len(v) for v in self.negitems.values()) + \ - len(self.pzero) + \ - sum(len(v) for v in self.positems.values()) - return total - - def __iter__(self): - gen_negs = ((i, priority) - for priority in sorted(self.negitems.keys()) - for i in reversed(self.negitems[priority])) - gen_zeros = ((item,0) for item in self.pzero) - gen_pos = ((i, priority) - for priority in sorted(self.positems.keys()) - for i in reversed(self.positems[priority])) - return chain(gen_negs, gen_zeros, gen_pos) - - - def __nonzero__(self): - return bool(self.negitems or self.pzero or self.positems) - -class PriorityQueue4b(object): - """deque+defaultdict+deque+cache""" - - def __init__(self, size=1): - self.negitems = defaultdict(deque) - self.pzero = deque() - self.positems = defaultdict(deque) - self._sort_neg = False - self._sort_pos = False - self._cached_sorted_negitems = [] - self._cached_sorted_positems = [] - - def push(self, item, priority=0): - if priority == 0: - self.pzero.appendleft(item) - elif priority < 0: - if priority not in self.negitems: - self._sort_neg = True - self.negitems[priority].appendleft(item) - else: - if priority not in self.positems: - self._sort_pos = True - self.positems[priority].appendleft(item) - - def pop(self): - if self.negitems: - if self._sort_neg: - priorities = self.negitems.keys() - priorities.sort() - self._cached_sorted_negitems = priorities - self._sort_neg = False - else: - priorities = self._cached_sorted_negitems - - for priority in priorities: - deq = self.negitems[priority] - if deq: - t = (deq.pop(), priority) - if not deq: - self._sort_neg = True - del self.negitems[priority] - return t - elif self.pzero: - return (self.pzero.pop(), 0) - else: - if self._sort_pos: - priorities = self.positems.keys() - priorities.sort() - self._cached_sorted_positems = priorities - self._sort_pos = False - else: - priorities = self._cached_sorted_positems - - priorities = self.positems.keys() - priorities.sort() - for priority in priorities: - deq = self.positems[priority] - if deq: - t = (deq.pop(), priority) - if not deq: - self._sort_pos = True - del self.positems[priority] - return t - raise IndexError("pop from an empty queue") - - def __len__(self): - total = sum(len(v) for v in self.negitems.values()) + \ - len(self.pzero) + \ - sum(len(v) for v in self.positems.values()) - return total - - def __iter__(self): - gen_negs = ((i, priority) - for priority in sorted(self.negitems.keys()) - for i in reversed(self.negitems[priority])) - gen_zeros = ((item,0) for item in self.pzero) - gen_pos = ((i, priority) - for priority in sorted(self.positems.keys()) - for i in reversed(self.positems[priority])) - return chain(gen_negs, gen_zeros, gen_pos) - - - def __nonzero__(self): - return bool(self.negitems or self.pzero or self.positems) - -#------------------------------------------------------------------------------ - -class PriorityQueue5(object): - """list+deque""" - - def __init__(self, size=1): - # preallocate deques for a fixed number of priorities - size = size if size % 2 else size + 1 - self.zero = size // 2 - self.priolist = [deque() for _ in range(size)] - - def push(self, item, priority=0): - self.priolist[priority + self.zero].appendleft(item) - - def pop(self): - for prio, queue in enumerate(self.priolist): - if len(queue): - final = prio - self.zero - return (queue.pop(), final) - - raise IndexError("pop from an empty queue") - - def __len__(self): - return sum(len(v) for v in self.priolist) - - def __iter__(self): - for prio, queue in enumerate(self.priolist): - final = prio - self.zero - for i in reversed(queue): - yield (i, final) - - def __nonzero__(self): - return any(d for d in self.priolist) - -class PriorityQueue5b(PriorityQueue5): - """list+deque+cache""" - index = 0 - - def push(self, item, priority=0): - i = priority + self.zero - self.priolist[i].appendleft(item) - self.index = min(self.index, i) - - def pop(self): - cached = self.priolist[self.index] - if cached: - return (cached.pop(), self.index - self.zero) - - for prio, queue in enumerate(self.priolist[self.index:]): - if queue: - self.index += prio - return (queue.pop(), self.index - self.zero) - - raise IndexError("pop from an empty queue") - - -from itertools import islice -class PriorityQueue5c(PriorityQueue5b): - """list+deque+cache+islice""" - - def pop(self): - cached = self.priolist[self.index] - if cached: - return (cached.pop(), self.index - self.zero) - - for prio, queue in enumerate(islice(self.priolist, self.index, None), self.index): - if queue: - self.index = prio - return (queue.pop(), prio - self.zero) - - raise IndexError("pop from an empty queue") - -#------------------------------------------------------------------------------ - -__all__ = [name for name in globals().keys() if name.startswith('PriorityQueue')] diff --git a/profiling/priorityqueue/run.py b/profiling/priorityqueue/run.py deleted file mode 100644 index 2cc5f9ed2..000000000 --- a/profiling/priorityqueue/run.py +++ /dev/null @@ -1,157 +0,0 @@ -from __future__ import with_statement -import os -import timeit -import random -from optparse import OptionParser -from tempfile import mktemp -from pq_classes import * - -TESTID = os.getpid() - -TESTCASES = ( - ("heapq", PriorityQueue1), - #("heapq+int", PriorityQueue1b), - ("dict+deque", PriorityQueue2), - ("deque+heapq", PriorityQueue3), - #("deque+heapq+int", PriorityQueue3b), - ("deque+defaultdict+deque", PriorityQueue4), - ("deque+defaultdict+deque+cache", PriorityQueue4b), - #('list+deque', PriorityQueue5), - ('list+deque+cache', PriorityQueue5b), - #('list+deque+cache+islice', PriorityQueue5c), - ) - - -stmt_fmt = """ -for n, prio in enumerate(randomprio): - q.push(n, prio) - -try: - while True: - q.pop() -except IndexError: - pass -""" - -setup_fmt = """ -from collections import deque -from __main__ import %(PriorityClass)s as PriorityQueue -q = PriorityQueue(%(priorities)i) - -randomprio = deque() -for line in open('%(samplefile)s'): - prio = int(line.strip()) - randomprio.append(prio) -""" - - -def _distribution(priorities, distribution): - half = priorities // 2 - prio = -priorities - while not (-half <= prio <= half): - prio = round(distribution()) - return min(max(prio, -half), half) - -def normal_priority(priorities): - sigma = priorities / 4.0 - dist = lambda: random.normalvariate(mu=0, sigma=sigma) - return _distribution(priorities, dist) - -def gauss_priority(priorities): - sigma = priorities / 4.0 - dist = lambda: random.gauss(mu=0, sigma=sigma) - return _distribution(priorities, dist) - -def triangular_priority(priorities): - half = priorities // 2 - return random.triangular(-half-1, half+1, 0) - -def uniform_priority(priorities): - return int(random.random() * priorities) - (priorities / 2) - - -PRIORITY_DISTRIBUTIONS = { - 'uniform': uniform_priority, - 'normal': normal_priority, - 'gauss': gauss_priority, - 'triangular': triangular_priority, - } - - -def gen_samples(count, priorities, priority_distribution=uniform_priority): - fn = '/tmp/pq-%i-%i-%i' % (TESTID, priorities, count) - - with open(fn, 'w') as samplefile: - for n in xrange(count): - prio = priority_distribution(priorities) - samplefile.write('%i\n' % prio) - return fn - -def runtests(pushpops=50*1000, times=30, priorities=1, samplefile=None, priority_distribution=uniform_priority): - samplefile = samplefile or gen_samples(pushpops, priorities, priority_distribution) - - print "\n== With %s priorities (%s) ==\n" % (priorities, samplefile) - print "pushpops = %s, times = %s" % (pushpops, times) - - - stmt = stmt_fmt - for name, cls in TESTCASES: - setup = setup_fmt % { - 'PriorityClass': cls.__name__, - 'priorities': priorities, - 'samplefile': samplefile, - } - t = timeit.Timer(stmt, setup) - print "%s implementation: %s" % (name, t.timeit(number=times)) - - -if __name__ == '__main__': - o = OptionParser() - o.add_option('-n', '--samples-count', type='int', default=50000, metavar='NUMBER', - help='The max number or samples to generate') - o.add_option('-r', '--retry-times', type='int', default=30, metavar='NUMBER', - help='the times to retry each test') - o.add_option('-s', '--samplefile', default=None, metavar='FILENAME', - help='load samples from file, default: use sample generator') - o.add_option('-p', '--priorities', default='1,3,5,10,100', metavar='CSV_PRIOLIST', - help='a comma separated list of priorities to test') - o.add_option('-d', '--priority-distribution', default='uniform', metavar='DISTRIBUTION', - help='distribution used for random priority generator, default: uniform. possibles: %s' \ - % ','.join(PRIORITY_DISTRIBUTIONS.keys())) - - opt, args = o.parse_args() - - priolist = map(int, opt.priorities.split(',')) - distribution = PRIORITY_DISTRIBUTIONS[opt.priority_distribution] - for prio in priolist: - runtests(pushpops=opt.samples_count, priorities=prio, times=opt.retry_times, - samplefile=opt.samplefile, priority_distribution=distribution) - -# Results (in seconds, on an intel core2 2.16ghz): -# == Without priorities == - -# pushpops = 50000, times = 30 -# heapq implementation: 7.7959010601 -# dict+deque implementation: 5.6420109272 -# deque+heapq implementation: 3.57563900948 - -# == With 5 priorities == - -# pushpops = 50000, times = 30 -# heapq implementation: 9.83902192116 -# dict+deque implementation: 9.21094298363 -# deque+heapq implementation: 9.05321097374 - -# == With 10 priorities == - -# pushpops = 50000, times = 30 -# heapq implementation: 9.97831392288 -# dict+deque implementation: 11.9721341133 -# deque+heapq implementation: 9.79048800468 - -# == With 100 priorities == - -# pushpops = 50000, times = 30 -# heapq implementation: 10.4782910347 -# dict+deque implementation: 64.6989660263 -# deque+heapq implementation: 10.858932972 diff --git a/profiling/priorityqueue/test_cases.py b/profiling/priorityqueue/test_cases.py deleted file mode 100644 index 4b87f3e0c..000000000 --- a/profiling/priorityqueue/test_cases.py +++ /dev/null @@ -1,71 +0,0 @@ -import sys -import unittest -from pq_classes import * - -# (ITEM, PRIORITY) -INPUT = [(1, -5), (30, -1), (80, -3), (4, 1), (6, 3), (20, 0), (50, -1)] -OUTPUT = [(1, -5), (80, -3), (30, -1), (50, -1), (20, 0), (4, 1), (6, 3)] - -PRIOSIZE = reduce(max, (abs(i[1]) for i in INPUT + OUTPUT)) * 2 - -class TestPriorityQueue(object): - - def test_popping(self): - pq = self.PriorityQueue(PRIOSIZE) - for item, pr in INPUT: - pq.push(item, pr) - l = [] - while pq: - l.append(pq.pop()) - self.assertEquals(l, OUTPUT) - - def test_iter(self): - pq = self.PriorityQueue(PRIOSIZE) - for item, pr in INPUT: - pq.push(item, pr) - result = [x for x in pq] - self.assertEquals(result, OUTPUT) - - def test_nonzero(self): - pq = self.PriorityQueue(PRIOSIZE) - pq.push(80, -1) - pq.push(20, 0) - pq.push(30, 1) - - pq.pop() - self.assertEquals(bool(pq), True) - pq.pop() - self.assertEquals(bool(pq), True) - pq.pop() - self.assertEquals(bool(pq), False) - - def test_len(self): - pq = self.PriorityQueue(PRIOSIZE) - pq.push(80, -1) - pq.push(20, 0) - pq.push(30, 1) - - self.assertEquals(len(pq), 3) - pq.pop() - self.assertEquals(len(pq), 2) - pq.pop() - self.assertEquals(len(pq), 1) - pq.pop() - self.assertEquals(len(pq), 0) - - def test_indexerror(self): - pq = self.PriorityQueue(PRIOSIZE) - self.assertRaises(IndexError, pq.pop) - - -# automatically test any PriorityQueueN class defined at pq_classes -prioclasses = [(name, cls) for name, cls in globals().items() if name.startswith('PriorityQueue')] -for name, cls in prioclasses: - testname = '%sTestCase' % name - testcase = type(testname, (TestPriorityQueue, unittest.TestCase), {'PriorityQueue': cls}) - setattr(sys.modules[__name__], testname, testcase) - - -if __name__ == '__main__': - print "\n== Unit testing for every implementation ==" - unittest.main()