diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py new file mode 100755 index 000000000..b2c824bb3 --- /dev/null +++ b/extras/qps-bench-server.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +from time import time +from collections import deque +from twisted.web.server import Site, NOT_DONE_YET +from twisted.web.resource import Resource +from twisted.internet import reactor + + +class Root(Resource): + + def __init__(self): + Resource.__init__(self) + self.concurrent = 0 + self.tail = deque(maxlen=100) + self._reset_stats() + + def _reset_stats(self): + self.tail.clear() + self.start = self.lastmark = self.lasttime = time() + + def getChild(self, request, name): + return self + + def render(self, request): + now = time() + delta = now - self.lasttime + + # reset stats on high iter-request times caused by client restarts + if delta > 3: # seconds + self._reset_stats() + return '' + + self.tail.appendleft(delta) + self.lasttime = now + self.concurrent += 1 + + if now - self.lastmark >= 3: + self.lastmark = now + qps = len(self.tail) / sum(self.tail) + print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) + + if 'sleep' in request.args: + sleep = float(request.args['sleep'][0]) + reactor.callLater(sleep, self._finish, request) + return NOT_DONE_YET + + self.concurrent -= 1 + return '' + + def _finish(self, request): + self.concurrent -= 1 + if not request.finished and not request._disconnected: + request.finish() + + +root = Root() +factory = Site(root) +reactor.listenTCP(8880, factory) +reactor.run() diff --git a/extras/qpsclient.py b/extras/qpsclient.py new file mode 100644 index 000000000..53783b52a --- /dev/null +++ b/extras/qpsclient.py @@ -0,0 +1,42 @@ +""" +A spider that generate light requests to meassure QPS troughput + +usage: + + scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a sleep=0.3 + +""" + +from scrapy.spider import BaseSpider +from scrapy.http import Request + + +class QPSSpider(BaseSpider): + + name = 'qps' + benchurl = 'http://localhost:8880/' + + + # Max concurrency is limited by global CONCURRENT_REQUESTS setting + max_concurrent_requests = 8 + # Requests per second goal + qps = None # same as: 1 / download_delay + # time in seconds to delay server responses + sleep = None + + def __init__(self, *a, **kw): + super(QPSSpider, self).__init__(*a, **kw) + if self.qps is not None: + self.qps = float(self.qps) + self.download_delay = 1 / self.qps + + def start_requests(self): + url = self.benchurl + if self.sleep is not None: + url += '?sleep={0}'.format(self.sleep) + + while True: + yield Request(url, dont_filter=True) + + def parse(self, response): + pass