mirror of https://github.com/scrapy/scrapy.git
removed obsolete profiling/ dir
This commit is contained in:
Pablo Hoffman
parent
f354a49d0f
commit
4d1e01a4d4
|
|
@ -1,370 +0,0 @@
|
|||
from collections import deque, defaultdict
|
||||
from heapq import heappush, heappop
|
||||
import time
|
||||
from itertools import chain
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue1(object):
|
||||
"""heapq
|
||||
|
||||
A simple priority queue
|
||||
"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.items = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
heappush(self.items, (priority, time.time(), item))
|
||||
|
||||
def pop(self):
|
||||
priority, _, item = heappop(self.items)
|
||||
return item, priority
|
||||
|
||||
def __len__(self):
|
||||
return len(self.items)
|
||||
|
||||
def __iter__(self):
|
||||
return ((item, priority) for priority, _, item in self.items)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.items)
|
||||
|
||||
|
||||
class PriorityQueue1b(PriorityQueue1):
|
||||
"""heapq+int
|
||||
|
||||
A simple priority queue using incremental integer instead of time.time()
|
||||
"""
|
||||
time = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.time += 1
|
||||
heappush(self.items, (priority, time, item))
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue2(object):
|
||||
"""dict+deque
|
||||
|
||||
@author: Federico Feroldi <federico@cloudify.me>
|
||||
"""
|
||||
def __init__(self, size=1):
|
||||
self.items = defaultdict(deque)
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.items[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
priorities = self.items.keys()
|
||||
priorities.sort()
|
||||
|
||||
for priority in priorities:
|
||||
if len(self.items[priority]) > 0:
|
||||
return (self.items[priority].pop(), priority)
|
||||
|
||||
raise IndexError
|
||||
|
||||
def __len__(self):
|
||||
totlen = 0
|
||||
for q in self.items.values():
|
||||
totlen += len(q)
|
||||
return totlen
|
||||
|
||||
def __iter__(self):
|
||||
priorities = self.items.keys()
|
||||
priorities.sort()
|
||||
|
||||
for priority in priorities:
|
||||
for i in self.items[priority]:
|
||||
yield (i, priority)
|
||||
|
||||
def __nonzero__(self):
|
||||
for q in self.items.values():
|
||||
if len(q) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue3(object):
|
||||
"""deque+heapq"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = []
|
||||
self.pzero = deque()
|
||||
self.positems = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
heappush(self.negitems, (priority, time.time(), item))
|
||||
else:
|
||||
heappush(self.positems, (priority, time.time(), item))
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
priority, _, item = heappop(self.negitems)
|
||||
return item, priority
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
priority, _, item = heappop(self.positems)
|
||||
return item, priority
|
||||
|
||||
def __len__(self):
|
||||
return len(self.negitems) + len(self.pzero) + len(self.positems)
|
||||
|
||||
def __iter__(self):
|
||||
for priority, _, item in self.negitems:
|
||||
yield (item, priority)
|
||||
for item in self.pzero:
|
||||
yield (item, 0)
|
||||
for priority, _, item in self.positems:
|
||||
yield (item, priority)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems and self.pzero and self.positems)
|
||||
|
||||
|
||||
class PriorityQueue3b(PriorityQueue3):
|
||||
"""deque+heapq+int"""
|
||||
|
||||
left_time = 0
|
||||
right_time = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
self.left_time += 1
|
||||
heappush(self.negitems, (priority, self.left_time, item))
|
||||
else:
|
||||
self.right_time += 1
|
||||
heappush(self.positems, (priority, self.right_time, item))
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue4(object):
|
||||
"""deque+defaultdict+deque"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = defaultdict(deque)
|
||||
self.pzero = deque()
|
||||
self.positems = defaultdict(deque)
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
self.negitems[priority].appendleft(item)
|
||||
else:
|
||||
self.positems[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
priorities = self.negitems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.negitems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
del self.negitems[priority]
|
||||
return t
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.positems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
del self.positems[priority]
|
||||
return t
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
total = sum(len(v) for v in self.negitems.values()) + \
|
||||
len(self.pzero) + \
|
||||
sum(len(v) for v in self.positems.values())
|
||||
return total
|
||||
|
||||
def __iter__(self):
|
||||
gen_negs = ((i, priority)
|
||||
for priority in sorted(self.negitems.keys())
|
||||
for i in reversed(self.negitems[priority]))
|
||||
gen_zeros = ((item,0) for item in self.pzero)
|
||||
gen_pos = ((i, priority)
|
||||
for priority in sorted(self.positems.keys())
|
||||
for i in reversed(self.positems[priority]))
|
||||
return chain(gen_negs, gen_zeros, gen_pos)
|
||||
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems or self.pzero or self.positems)
|
||||
|
||||
class PriorityQueue4b(object):
|
||||
"""deque+defaultdict+deque+cache"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = defaultdict(deque)
|
||||
self.pzero = deque()
|
||||
self.positems = defaultdict(deque)
|
||||
self._sort_neg = False
|
||||
self._sort_pos = False
|
||||
self._cached_sorted_negitems = []
|
||||
self._cached_sorted_positems = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
if priority not in self.negitems:
|
||||
self._sort_neg = True
|
||||
self.negitems[priority].appendleft(item)
|
||||
else:
|
||||
if priority not in self.positems:
|
||||
self._sort_pos = True
|
||||
self.positems[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
if self._sort_neg:
|
||||
priorities = self.negitems.keys()
|
||||
priorities.sort()
|
||||
self._cached_sorted_negitems = priorities
|
||||
self._sort_neg = False
|
||||
else:
|
||||
priorities = self._cached_sorted_negitems
|
||||
|
||||
for priority in priorities:
|
||||
deq = self.negitems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
self._sort_neg = True
|
||||
del self.negitems[priority]
|
||||
return t
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
if self._sort_pos:
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
self._cached_sorted_positems = priorities
|
||||
self._sort_pos = False
|
||||
else:
|
||||
priorities = self._cached_sorted_positems
|
||||
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.positems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
self._sort_pos = True
|
||||
del self.positems[priority]
|
||||
return t
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
total = sum(len(v) for v in self.negitems.values()) + \
|
||||
len(self.pzero) + \
|
||||
sum(len(v) for v in self.positems.values())
|
||||
return total
|
||||
|
||||
def __iter__(self):
|
||||
gen_negs = ((i, priority)
|
||||
for priority in sorted(self.negitems.keys())
|
||||
for i in reversed(self.negitems[priority]))
|
||||
gen_zeros = ((item,0) for item in self.pzero)
|
||||
gen_pos = ((i, priority)
|
||||
for priority in sorted(self.positems.keys())
|
||||
for i in reversed(self.positems[priority]))
|
||||
return chain(gen_negs, gen_zeros, gen_pos)
|
||||
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems or self.pzero or self.positems)
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue5(object):
|
||||
"""list+deque"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
# preallocate deques for a fixed number of priorities
|
||||
size = size if size % 2 else size + 1
|
||||
self.zero = size // 2
|
||||
self.priolist = [deque() for _ in range(size)]
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.priolist[priority + self.zero].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
for prio, queue in enumerate(self.priolist):
|
||||
if len(queue):
|
||||
final = prio - self.zero
|
||||
return (queue.pop(), final)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
return sum(len(v) for v in self.priolist)
|
||||
|
||||
def __iter__(self):
|
||||
for prio, queue in enumerate(self.priolist):
|
||||
final = prio - self.zero
|
||||
for i in reversed(queue):
|
||||
yield (i, final)
|
||||
|
||||
def __nonzero__(self):
|
||||
return any(d for d in self.priolist)
|
||||
|
||||
class PriorityQueue5b(PriorityQueue5):
|
||||
"""list+deque+cache"""
|
||||
index = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
i = priority + self.zero
|
||||
self.priolist[i].appendleft(item)
|
||||
self.index = min(self.index, i)
|
||||
|
||||
def pop(self):
|
||||
cached = self.priolist[self.index]
|
||||
if cached:
|
||||
return (cached.pop(), self.index - self.zero)
|
||||
|
||||
for prio, queue in enumerate(self.priolist[self.index:]):
|
||||
if queue:
|
||||
self.index += prio
|
||||
return (queue.pop(), self.index - self.zero)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
|
||||
from itertools import islice
|
||||
class PriorityQueue5c(PriorityQueue5b):
|
||||
"""list+deque+cache+islice"""
|
||||
|
||||
def pop(self):
|
||||
cached = self.priolist[self.index]
|
||||
if cached:
|
||||
return (cached.pop(), self.index - self.zero)
|
||||
|
||||
for prio, queue in enumerate(islice(self.priolist, self.index, None), self.index):
|
||||
if queue:
|
||||
self.index = prio
|
||||
return (queue.pop(), prio - self.zero)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
__all__ = [name for name in globals().keys() if name.startswith('PriorityQueue')]
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
from __future__ import with_statement
|
||||
import os
|
||||
import timeit
|
||||
import random
|
||||
from optparse import OptionParser
|
||||
from tempfile import mktemp
|
||||
from pq_classes import *
|
||||
|
||||
TESTID = os.getpid()
|
||||
|
||||
TESTCASES = (
|
||||
("heapq", PriorityQueue1),
|
||||
#("heapq+int", PriorityQueue1b),
|
||||
("dict+deque", PriorityQueue2),
|
||||
("deque+heapq", PriorityQueue3),
|
||||
#("deque+heapq+int", PriorityQueue3b),
|
||||
("deque+defaultdict+deque", PriorityQueue4),
|
||||
("deque+defaultdict+deque+cache", PriorityQueue4b),
|
||||
#('list+deque', PriorityQueue5),
|
||||
('list+deque+cache', PriorityQueue5b),
|
||||
#('list+deque+cache+islice', PriorityQueue5c),
|
||||
)
|
||||
|
||||
|
||||
stmt_fmt = """
|
||||
for n, prio in enumerate(randomprio):
|
||||
q.push(n, prio)
|
||||
|
||||
try:
|
||||
while True:
|
||||
q.pop()
|
||||
except IndexError:
|
||||
pass
|
||||
"""
|
||||
|
||||
setup_fmt = """
|
||||
from collections import deque
|
||||
from __main__ import %(PriorityClass)s as PriorityQueue
|
||||
q = PriorityQueue(%(priorities)i)
|
||||
|
||||
randomprio = deque()
|
||||
for line in open('%(samplefile)s'):
|
||||
prio = int(line.strip())
|
||||
randomprio.append(prio)
|
||||
"""
|
||||
|
||||
|
||||
def _distribution(priorities, distribution):
|
||||
half = priorities // 2
|
||||
prio = -priorities
|
||||
while not (-half <= prio <= half):
|
||||
prio = round(distribution())
|
||||
return min(max(prio, -half), half)
|
||||
|
||||
def normal_priority(priorities):
|
||||
sigma = priorities / 4.0
|
||||
dist = lambda: random.normalvariate(mu=0, sigma=sigma)
|
||||
return _distribution(priorities, dist)
|
||||
|
||||
def gauss_priority(priorities):
|
||||
sigma = priorities / 4.0
|
||||
dist = lambda: random.gauss(mu=0, sigma=sigma)
|
||||
return _distribution(priorities, dist)
|
||||
|
||||
def triangular_priority(priorities):
|
||||
half = priorities // 2
|
||||
return random.triangular(-half-1, half+1, 0)
|
||||
|
||||
def uniform_priority(priorities):
|
||||
return int(random.random() * priorities) - (priorities / 2)
|
||||
|
||||
|
||||
PRIORITY_DISTRIBUTIONS = {
|
||||
'uniform': uniform_priority,
|
||||
'normal': normal_priority,
|
||||
'gauss': gauss_priority,
|
||||
'triangular': triangular_priority,
|
||||
}
|
||||
|
||||
|
||||
def gen_samples(count, priorities, priority_distribution=uniform_priority):
|
||||
fn = '/tmp/pq-%i-%i-%i' % (TESTID, priorities, count)
|
||||
|
||||
with open(fn, 'w') as samplefile:
|
||||
for n in xrange(count):
|
||||
prio = priority_distribution(priorities)
|
||||
samplefile.write('%i\n' % prio)
|
||||
return fn
|
||||
|
||||
def runtests(pushpops=50*1000, times=30, priorities=1, samplefile=None, priority_distribution=uniform_priority):
|
||||
samplefile = samplefile or gen_samples(pushpops, priorities, priority_distribution)
|
||||
|
||||
print "\n== With %s priorities (%s) ==\n" % (priorities, samplefile)
|
||||
print "pushpops = %s, times = %s" % (pushpops, times)
|
||||
|
||||
|
||||
stmt = stmt_fmt
|
||||
for name, cls in TESTCASES:
|
||||
setup = setup_fmt % {
|
||||
'PriorityClass': cls.__name__,
|
||||
'priorities': priorities,
|
||||
'samplefile': samplefile,
|
||||
}
|
||||
t = timeit.Timer(stmt, setup)
|
||||
print "%s implementation: %s" % (name, t.timeit(number=times))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
o = OptionParser()
|
||||
o.add_option('-n', '--samples-count', type='int', default=50000, metavar='NUMBER',
|
||||
help='The max number or samples to generate')
|
||||
o.add_option('-r', '--retry-times', type='int', default=30, metavar='NUMBER',
|
||||
help='the times to retry each test')
|
||||
o.add_option('-s', '--samplefile', default=None, metavar='FILENAME',
|
||||
help='load samples from file, default: use sample generator')
|
||||
o.add_option('-p', '--priorities', default='1,3,5,10,100', metavar='CSV_PRIOLIST',
|
||||
help='a comma separated list of priorities to test')
|
||||
o.add_option('-d', '--priority-distribution', default='uniform', metavar='DISTRIBUTION',
|
||||
help='distribution used for random priority generator, default: uniform. possibles: %s' \
|
||||
% ','.join(PRIORITY_DISTRIBUTIONS.keys()))
|
||||
|
||||
opt, args = o.parse_args()
|
||||
|
||||
priolist = map(int, opt.priorities.split(','))
|
||||
distribution = PRIORITY_DISTRIBUTIONS[opt.priority_distribution]
|
||||
for prio in priolist:
|
||||
runtests(pushpops=opt.samples_count, priorities=prio, times=opt.retry_times,
|
||||
samplefile=opt.samplefile, priority_distribution=distribution)
|
||||
|
||||
# Results (in seconds, on an intel core2 2.16ghz):
|
||||
# == Without priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 7.7959010601
|
||||
# dict+deque implementation: 5.6420109272
|
||||
# deque+heapq implementation: 3.57563900948
|
||||
|
||||
# == With 5 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 9.83902192116
|
||||
# dict+deque implementation: 9.21094298363
|
||||
# deque+heapq implementation: 9.05321097374
|
||||
|
||||
# == With 10 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 9.97831392288
|
||||
# dict+deque implementation: 11.9721341133
|
||||
# deque+heapq implementation: 9.79048800468
|
||||
|
||||
# == With 100 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 10.4782910347
|
||||
# dict+deque implementation: 64.6989660263
|
||||
# deque+heapq implementation: 10.858932972
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
import sys
|
||||
import unittest
|
||||
from pq_classes import *
|
||||
|
||||
# (ITEM, PRIORITY)
|
||||
INPUT = [(1, -5), (30, -1), (80, -3), (4, 1), (6, 3), (20, 0), (50, -1)]
|
||||
OUTPUT = [(1, -5), (80, -3), (30, -1), (50, -1), (20, 0), (4, 1), (6, 3)]
|
||||
|
||||
PRIOSIZE = reduce(max, (abs(i[1]) for i in INPUT + OUTPUT)) * 2
|
||||
|
||||
class TestPriorityQueue(object):
|
||||
|
||||
def test_popping(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
for item, pr in INPUT:
|
||||
pq.push(item, pr)
|
||||
l = []
|
||||
while pq:
|
||||
l.append(pq.pop())
|
||||
self.assertEquals(l, OUTPUT)
|
||||
|
||||
def test_iter(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
for item, pr in INPUT:
|
||||
pq.push(item, pr)
|
||||
result = [x for x in pq]
|
||||
self.assertEquals(result, OUTPUT)
|
||||
|
||||
def test_nonzero(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
pq.push(80, -1)
|
||||
pq.push(20, 0)
|
||||
pq.push(30, 1)
|
||||
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), True)
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), True)
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), False)
|
||||
|
||||
def test_len(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
pq.push(80, -1)
|
||||
pq.push(20, 0)
|
||||
pq.push(30, 1)
|
||||
|
||||
self.assertEquals(len(pq), 3)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 2)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 1)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 0)
|
||||
|
||||
def test_indexerror(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
self.assertRaises(IndexError, pq.pop)
|
||||
|
||||
|
||||
# automatically test any PriorityQueueN class defined at pq_classes
|
||||
prioclasses = [(name, cls) for name, cls in globals().items() if name.startswith('PriorityQueue')]
|
||||
for name, cls in prioclasses:
|
||||
testname = '%sTestCase' % name
|
||||
testcase = type(testname, (TestPriorityQueue, unittest.TestCase), {'PriorityQueue': cls})
|
||||
setattr(sys.modules[__name__], testname, testcase)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print "\n== Unit testing for every implementation =="
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue