diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 4bfea6e18..18934f976 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,6 +1,13 @@
+import sys
+import time
+import subprocess
+
+from six.moves.urllib.parse import urlencode
+
+import scrapy
from scrapy.command import ScrapyCommand
-from tests.spiders import FollowAllSpider
-from tests.mockserver import MockServer
+from scrapy.contrib.linkextractors import LinkExtractor
+
class Command(ScrapyCommand):
@@ -14,8 +21,41 @@ class Command(ScrapyCommand):
return "Run quick benchmark test"
def run(self, args, opts):
- with MockServer():
- spider = FollowAllSpider(total=100000)
+ with _BenchServer():
+ spider = _BenchSpider(total=100000)
crawler = self.crawler_process.create_crawler()
crawler.crawl(spider)
self.crawler_process.start()
+
+
+class _BenchServer(object):
+
+ def __enter__(self):
+ from scrapy.utils.test import get_testenv
+ pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
+ self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
+ env=get_testenv())
+ self.proc.stdout.readline()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.proc.kill()
+ self.proc.wait()
+ time.sleep(0.2)
+
+
+class _BenchSpider(scrapy.Spider):
+ """A spider that follows all links"""
+ name = 'follow'
+ total = 10000
+ show = 20
+ baseurl = 'http://localhost:8998'
+ link_extractor = LinkExtractor()
+
+ def start_requests(self):
+ qargs = {'total': self.total, 'show': self.show}
+ url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
+ return [scrapy.Request(url, dont_filter=True)]
+
+ def parse(self, response):
+ for link in self.link_extractor.extract_links(response):
+ yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
new file mode 100644
index 000000000..4385d72a9
--- /dev/null
+++ b/scrapy/utils/benchserver.py
@@ -0,0 +1,44 @@
+import random
+from six.moves.urllib.parse import urlencode
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+from twisted.internet import reactor
+
+
+class Root(Resource):
+
+ isLeaf = True
+
+ def getChild(self, name, request):
+ return self
+
+ def render(self, request):
+ total = _getarg(request, 'total', 100, int)
+ show = _getarg(request, 'show', 10, int)
+ nlist = [random.randint(1, total) for _ in range(show)]
+ request.write("")
+ args = request.args.copy()
+ for nl in nlist:
+ args['n'] = nl
+ argstr = urlencode(args, doseq=True)
+ request.write("follow {1}
"
+ .format(argstr, nl))
+ request.write("")
+ return ''
+
+
+def _getarg(request, name, default=None, type=str):
+ return type(request.args[name][0]) \
+ if name in request.args else default
+
+
+if __name__ == '__main__':
+ root = Root()
+ factory = Site(root)
+ httpPort = reactor.listenTCP(8998, Site(root))
+
+ def _print_listening():
+ httpHost = httpPort.getHost()
+ print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port))
+ reactor.callWhenRunning(_print_listening)
+ reactor.run()