Fix bench command

2014-07-31 12:21:17 -03:00 · 2014-07-31 12:21:17 -03:00 · 7a41ef417d
parent aaf531c7dc
commit 7a41ef417d
2 changed files with 88 additions and 4 deletions
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@ -1,6 +1,13 @@
+import sys
+import time
+import subprocess
+
+from six.moves.urllib.parse import urlencode
+
+import scrapy
 from scrapy.command import ScrapyCommand
-from tests.spiders import FollowAllSpider
-from tests.mockserver import MockServer
+from scrapy.contrib.linkextractors import LinkExtractor
+

 class Command(ScrapyCommand):

@ -14,8 +21,41 @@ class Command(ScrapyCommand):
        return "Run quick benchmark test"

    def run(self, args, opts):
-        with MockServer():
-            spider = FollowAllSpider(total=100000)
+        with _BenchServer():
+            spider = _BenchSpider(total=100000)
            crawler = self.crawler_process.create_crawler()
            crawler.crawl(spider)
            self.crawler_process.start()
+
+
+class _BenchServer(object):
+
+    def __enter__(self):
+        from scrapy.utils.test import get_testenv
+        pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
+        self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
+                                     env=get_testenv())
+        self.proc.stdout.readline()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
+
+class _BenchSpider(scrapy.Spider):
+    """A spider that follows all links"""
+    name = 'follow'
+    total = 10000
+    show = 20
+    baseurl = 'http://localhost:8998'
+    link_extractor = LinkExtractor()
+
+    def start_requests(self):
+        qargs = {'total': self.total, 'show': self.show}
+        url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
+        return [scrapy.Request(url, dont_filter=True)]
+
+    def parse(self, response):
+        for link in self.link_extractor.extract_links(response):
+            yield scrapy.Request(link.url, callback=self.parse)
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@ -0,0 +1,44 @@
+import random
+from six.moves.urllib.parse import urlencode
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+from twisted.internet import reactor
+
+
+class Root(Resource):
+
+    isLeaf = True
+
+    def getChild(self, name, request):
+        return self
+
+    def render(self, request):
+        total = _getarg(request, 'total', 100, int)
+        show = _getarg(request, 'show', 10, int)
+        nlist = [random.randint(1, total) for _ in range(show)]
+        request.write("<html><head></head><body>")
+        args = request.args.copy()
+        for nl in nlist:
+            args['n'] = nl
+            argstr = urlencode(args, doseq=True)
+            request.write("<a href='/follow?{0}'>follow {1}</a><br>"
+                          .format(argstr, nl))
+        request.write("</body></html>")
+        return ''
+
+
+def _getarg(request, name, default=None, type=str):
+    return type(request.args[name][0]) \
+        if name in request.args else default
+
+
+if __name__ == '__main__':
+    root = Root()
+    factory = Site(root)
+    httpPort = reactor.listenTCP(8998, Site(root))
+
+    def _print_listening():
+        httpHost = httpPort.getHost()
+        print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port))
+    reactor.callWhenRunning(_print_listening)
+    reactor.run()