From d74b60051d6b4901c7c3058332a2d14c979b01d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Jul 2013 16:22:03 -0300 Subject: [PATCH] take in count response latencies when testing download delays --- scrapy/tests/mockserver.py | 57 +++++++++++++++++++++----------------- scrapy/tests/spiders.py | 6 ++-- scrapy/tests/test_crawl.py | 29 ++++++++++++++----- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 8b1cc09e8..4dde6c2a9 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -13,31 +13,7 @@ def getarg(request, name, default=None, type=str): return default -class Follow(Resource): - - isLeaf = True - - def render(self, request): - total = getarg(request, "total", 100, type=int) - show = getarg(request, "show", 1, type=int) - order = getarg(request, "order", "desc") - n = getarg(request, "n", total, type=int) - if order == "rand": - nlist = [random.randint(1, total) for _ in range(show)] - else: # order == "desc" - nlist = range(n, max(n - show, 0), -1) - - s = """ """ - args = request.args.copy() - for nl in nlist: - args["n"] = [str(nl)] - argstr = urllib.urlencode(args, doseq=True) - s += "follow %d
" % (argstr, nl) - s += """""" - return s - - -class DeferMixin(Resource): +class DeferMixin(object): def deferRequest(self, request, delay, f, *a, **kw): def _cancelrequest(_): @@ -49,6 +25,37 @@ class DeferMixin(Resource): return d +class Follow(DeferMixin, Resource): + + isLeaf = True + + def render(self, request): + total = getarg(request, "total", 100, type=int) + show = getarg(request, "show", 1, type=int) + order = getarg(request, "order", "desc") + maxlatency = getarg(request, "maxlatency", 0, type=float) + n = getarg(request, "n", total, type=int) + if order == "rand": + nlist = [random.randint(1, total) for _ in range(show)] + else: # order == "desc" + nlist = range(n, max(n - show, 0), -1) + + lag = random.random() * maxlatency + self.deferRequest(request, lag, self.renderRequest, request, nlist) + return NOT_DONE_YET + + def renderRequest(self, request, nlist): + s = """ """ + args = request.args.copy() + for nl in nlist: + args["n"] = [str(nl)] + argstr = urllib.urlencode(args, doseq=True) + s += "follow %d
" % (argstr, nl) + s += """""" + request.write(s) + request.finish() + + class Delay(DeferMixin, Resource): isLeaf = True diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index e1d06d85c..2d8dbd808 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking """ import time +from urllib import urlencode from scrapy.spider import BaseSpider from scrapy.http import Request @@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider): name = 'follow' link_extractor = SgmlLinkExtractor() - def __init__(self, total=10, show=20, order="rand", *args, **kwargs): + def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): super(FollowAllSpider, self).__init__(*args, **kwargs) self.urls_visited = [] self.times = [] - url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) + qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} + url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) self.start_urls = [url] def parse(self, response): diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 256f2d9b1..2a5fbdb62 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -11,6 +11,7 @@ def docrawl(spider, settings=None): crawler.crawl(spider) return crawler.start() + class CrawlTestCase(TestCase): def setUp(self): @@ -24,16 +25,30 @@ class CrawlTestCase(TestCase): def test_follow_all(self): spider = FollowAllSpider() yield docrawl(spider) - self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url + self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): - spider = FollowAllSpider() - yield docrawl(spider, {"DOWNLOAD_DELAY": 1}) - t = spider.times[0] - for t2 in spider.times[1:]: - self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t)) - t = t2 + # short to long delays + yield self._test_delay(0.2, False) + yield self._test_delay(1, False) + # randoms + yield self._test_delay(0.2, True) + yield self._test_delay(1, True) + + @defer.inlineCallbacks + def _test_delay(self, delay, randomize): + settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} + spider = FollowAllSpider(maxlatency=delay * 2) + yield docrawl(spider, settings) + t = spider.times + totaltime = t[-1] - t[0] + avgd = totaltime / (len(t) - 1) + tolerance = 0.6 if randomize else 0.2 + self.assertTrue(avgd > delay * (1 - tolerance), + "download delay too small: %s" % avgd) + self.assertTrue(avgd < delay * (1 + tolerance), + "download delay too big: %s" % avgd) @defer.inlineCallbacks def test_timeout_success(self):