QPS bench server and a useful spider spider to generate requests

extras/qps-bench-server.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+from time import time
+from collections import deque
+from twisted.web.server import Site, NOT_DONE_YET
+from twisted.web.resource import Resource
+from twisted.internet import reactor
+
+
+class Root(Resource):
+
+    def __init__(self):
+        Resource.__init__(self)
+        self.concurrent = 0
+        self.tail = deque(maxlen=100)
+        self._reset_stats()
+
+    def _reset_stats(self):
+        self.tail.clear()
+        self.start = self.lastmark = self.lasttime = time()
+
+    def getChild(self, request, name):
+        return self
+
+    def render(self, request):
+        now = time()
+        delta = now - self.lasttime
+
+        # reset stats on high iter-request times caused by client restarts
+        if delta > 3: # seconds
+            self._reset_stats()
+            return ''
+
+        self.tail.appendleft(delta)
+        self.lasttime = now
+        self.concurrent += 1
+
+        if now - self.lastmark >= 3:
+            self.lastmark = now
+            qps = len(self.tail) / sum(self.tail)
+            print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
+
+        if 'sleep' in request.args:
+            sleep = float(request.args['sleep'][0])
+            reactor.callLater(sleep, self._finish, request)
+            return NOT_DONE_YET
+
+        self.concurrent -= 1
+        return ''
+
+    def _finish(self, request):
+        self.concurrent -= 1
+        if not request.finished and not request._disconnected:
+            request.finish()
+
+
+root = Root()
+factory = Site(root)
+reactor.listenTCP(8880, factory)
+reactor.run()

extras/qpsclient.py

@@ -0,0 +1,42 @@
+"""
+A spider that generate light requests to meassure QPS troughput
+
+usage:
+
+    scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a sleep=0.3
+
+"""
+
+from scrapy.spider import BaseSpider
+from scrapy.http import Request
+
+
+class QPSSpider(BaseSpider):
+
+    name = 'qps'
+    benchurl = 'http://localhost:8880/'
+
+
+    # Max concurrency is limited by global CONCURRENT_REQUESTS setting
+    max_concurrent_requests = 8
+    # Requests per second goal
+    qps = None # same as: 1 / download_delay
+    # time in seconds to delay server responses
+    sleep = None 
+
+    def __init__(self, *a, **kw):
+        super(QPSSpider, self).__init__(*a, **kw)
+        if self.qps is not None:
+            self.qps = float(self.qps)
+            self.download_delay = 1 / self.qps
+
+    def start_requests(self):
+        url = self.benchurl
+        if self.sleep is not None:
+            url += '?sleep={0}'.format(self.sleep)
+
+        while True:
+            yield Request(url, dont_filter=True)
+
+    def parse(self, response):
+        pass