mirror of https://github.com/scrapy/scrapy.git
take in count response latencies when testing download delays
This commit is contained in:
Daniel Graña
parent
891fa980d2
commit
d74b60051d
|
|
@ -13,31 +13,7 @@ def getarg(request, name, default=None, type=str):
|
|||
return default
|
||||
|
||||
|
||||
class Follow(Resource):
|
||||
|
||||
isLeaf = True
|
||||
|
||||
def render(self, request):
|
||||
total = getarg(request, "total", 100, type=int)
|
||||
show = getarg(request, "show", 1, type=int)
|
||||
order = getarg(request, "order", "desc")
|
||||
n = getarg(request, "n", total, type=int)
|
||||
if order == "rand":
|
||||
nlist = [random.randint(1, total) for _ in range(show)]
|
||||
else: # order == "desc"
|
||||
nlist = range(n, max(n - show, 0), -1)
|
||||
|
||||
s = """<html> <head></head> <body>"""
|
||||
args = request.args.copy()
|
||||
for nl in nlist:
|
||||
args["n"] = [str(nl)]
|
||||
argstr = urllib.urlencode(args, doseq=True)
|
||||
s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
|
||||
s += """</body>"""
|
||||
return s
|
||||
|
||||
|
||||
class DeferMixin(Resource):
|
||||
class DeferMixin(object):
|
||||
|
||||
def deferRequest(self, request, delay, f, *a, **kw):
|
||||
def _cancelrequest(_):
|
||||
|
|
@ -49,6 +25,37 @@ class DeferMixin(Resource):
|
|||
return d
|
||||
|
||||
|
||||
class Follow(DeferMixin, Resource):
|
||||
|
||||
isLeaf = True
|
||||
|
||||
def render(self, request):
|
||||
total = getarg(request, "total", 100, type=int)
|
||||
show = getarg(request, "show", 1, type=int)
|
||||
order = getarg(request, "order", "desc")
|
||||
maxlatency = getarg(request, "maxlatency", 0, type=float)
|
||||
n = getarg(request, "n", total, type=int)
|
||||
if order == "rand":
|
||||
nlist = [random.randint(1, total) for _ in range(show)]
|
||||
else: # order == "desc"
|
||||
nlist = range(n, max(n - show, 0), -1)
|
||||
|
||||
lag = random.random() * maxlatency
|
||||
self.deferRequest(request, lag, self.renderRequest, request, nlist)
|
||||
return NOT_DONE_YET
|
||||
|
||||
def renderRequest(self, request, nlist):
|
||||
s = """<html> <head></head> <body>"""
|
||||
args = request.args.copy()
|
||||
for nl in nlist:
|
||||
args["n"] = [str(nl)]
|
||||
argstr = urllib.urlencode(args, doseq=True)
|
||||
s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
|
||||
s += """</body>"""
|
||||
request.write(s)
|
||||
request.finish()
|
||||
|
||||
|
||||
class Delay(DeferMixin, Resource):
|
||||
|
||||
isLeaf = True
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking
|
|||
"""
|
||||
|
||||
import time
|
||||
from urllib import urlencode
|
||||
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.http import Request
|
||||
|
|
@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider):
|
|||
name = 'follow'
|
||||
link_extractor = SgmlLinkExtractor()
|
||||
|
||||
def __init__(self, total=10, show=20, order="rand", *args, **kwargs):
|
||||
def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
|
||||
super(FollowAllSpider, self).__init__(*args, **kwargs)
|
||||
self.urls_visited = []
|
||||
self.times = []
|
||||
url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
|
||||
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
|
||||
url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
|
||||
self.start_urls = [url]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ def docrawl(spider, settings=None):
|
|||
crawler.crawl(spider)
|
||||
return crawler.start()
|
||||
|
||||
|
||||
class CrawlTestCase(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -24,16 +25,30 @@ class CrawlTestCase(TestCase):
|
|||
def test_follow_all(self):
|
||||
spider = FollowAllSpider()
|
||||
yield docrawl(spider)
|
||||
self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
|
||||
self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_delay(self):
|
||||
spider = FollowAllSpider()
|
||||
yield docrawl(spider, {"DOWNLOAD_DELAY": 1})
|
||||
t = spider.times[0]
|
||||
for t2 in spider.times[1:]:
|
||||
self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t))
|
||||
t = t2
|
||||
# short to long delays
|
||||
yield self._test_delay(0.2, False)
|
||||
yield self._test_delay(1, False)
|
||||
# randoms
|
||||
yield self._test_delay(0.2, True)
|
||||
yield self._test_delay(1, True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _test_delay(self, delay, randomize):
|
||||
settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
|
||||
spider = FollowAllSpider(maxlatency=delay * 2)
|
||||
yield docrawl(spider, settings)
|
||||
t = spider.times
|
||||
totaltime = t[-1] - t[0]
|
||||
avgd = totaltime / (len(t) - 1)
|
||||
tolerance = 0.6 if randomize else 0.2
|
||||
self.assertTrue(avgd > delay * (1 - tolerance),
|
||||
"download delay too small: %s" % avgd)
|
||||
self.assertTrue(avgd < delay * (1 + tolerance),
|
||||
"download delay too big: %s" % avgd)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_timeout_success(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue