diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py
index 8b1cc09e8..4dde6c2a9 100644
--- a/scrapy/tests/mockserver.py
+++ b/scrapy/tests/mockserver.py
@@ -13,31 +13,7 @@ def getarg(request, name, default=None, type=str):
return default
-class Follow(Resource):
-
- isLeaf = True
-
- def render(self, request):
- total = getarg(request, "total", 100, type=int)
- show = getarg(request, "show", 1, type=int)
- order = getarg(request, "order", "desc")
- n = getarg(request, "n", total, type=int)
- if order == "rand":
- nlist = [random.randint(1, total) for _ in range(show)]
- else: # order == "desc"
- nlist = range(n, max(n - show, 0), -1)
-
- s = """ """
- args = request.args.copy()
- for nl in nlist:
- args["n"] = [str(nl)]
- argstr = urllib.urlencode(args, doseq=True)
- s += "follow %d
" % (argstr, nl)
- s += """"""
- return s
-
-
-class DeferMixin(Resource):
+class DeferMixin(object):
def deferRequest(self, request, delay, f, *a, **kw):
def _cancelrequest(_):
@@ -49,6 +25,37 @@ class DeferMixin(Resource):
return d
+class Follow(DeferMixin, Resource):
+
+ isLeaf = True
+
+ def render(self, request):
+ total = getarg(request, "total", 100, type=int)
+ show = getarg(request, "show", 1, type=int)
+ order = getarg(request, "order", "desc")
+ maxlatency = getarg(request, "maxlatency", 0, type=float)
+ n = getarg(request, "n", total, type=int)
+ if order == "rand":
+ nlist = [random.randint(1, total) for _ in range(show)]
+ else: # order == "desc"
+ nlist = range(n, max(n - show, 0), -1)
+
+ lag = random.random() * maxlatency
+ self.deferRequest(request, lag, self.renderRequest, request, nlist)
+ return NOT_DONE_YET
+
+ def renderRequest(self, request, nlist):
+ s = """ """
+ args = request.args.copy()
+ for nl in nlist:
+ args["n"] = [str(nl)]
+ argstr = urllib.urlencode(args, doseq=True)
+ s += "follow %d
" % (argstr, nl)
+ s += """"""
+ request.write(s)
+ request.finish()
+
+
class Delay(DeferMixin, Resource):
isLeaf = True
diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
index e1d06d85c..2d8dbd808 100644
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking
"""
import time
+from urllib import urlencode
from scrapy.spider import BaseSpider
from scrapy.http import Request
@@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider):
name = 'follow'
link_extractor = SgmlLinkExtractor()
- def __init__(self, total=10, show=20, order="rand", *args, **kwargs):
+ def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
super(FollowAllSpider, self).__init__(*args, **kwargs)
self.urls_visited = []
self.times = []
- url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
+ qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
+ url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
self.start_urls = [url]
def parse(self, response):
diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py
index 256f2d9b1..2a5fbdb62 100644
--- a/scrapy/tests/test_crawl.py
+++ b/scrapy/tests/test_crawl.py
@@ -11,6 +11,7 @@ def docrawl(spider, settings=None):
crawler.crawl(spider)
return crawler.start()
+
class CrawlTestCase(TestCase):
def setUp(self):
@@ -24,16 +25,30 @@ class CrawlTestCase(TestCase):
def test_follow_all(self):
spider = FollowAllSpider()
yield docrawl(spider)
- self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
+ self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
@defer.inlineCallbacks
def test_delay(self):
- spider = FollowAllSpider()
- yield docrawl(spider, {"DOWNLOAD_DELAY": 1})
- t = spider.times[0]
- for t2 in spider.times[1:]:
- self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t))
- t = t2
+ # short to long delays
+ yield self._test_delay(0.2, False)
+ yield self._test_delay(1, False)
+ # randoms
+ yield self._test_delay(0.2, True)
+ yield self._test_delay(1, True)
+
+ @defer.inlineCallbacks
+ def _test_delay(self, delay, randomize):
+ settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+ spider = FollowAllSpider(maxlatency=delay * 2)
+ yield docrawl(spider, settings)
+ t = spider.times
+ totaltime = t[-1] - t[0]
+ avgd = totaltime / (len(t) - 1)
+ tolerance = 0.6 if randomize else 0.2
+ self.assertTrue(avgd > delay * (1 - tolerance),
+ "download delay too small: %s" % avgd)
+ self.assertTrue(avgd < delay * (1 + tolerance),
+ "download delay too big: %s" % avgd)
@defer.inlineCallbacks
def test_timeout_success(self):