added mock server test for DOWNLOAD_TIMEOUT

scrapy/tests/mockserver.py

@@ -1,8 +1,9 @@
 import json, random, urllib
 from time import time
-from twisted.web.server import Site
+from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.internet import reactor
+from twisted.internet.task import deferLater
 
 
 _id = lambda x: x
@@ -52,6 +53,20 @@ class Follow(Resource):
         s += """</body>"""
         return s
 
+class Delay(Resource):
+
+    isLeaf = True
+
+    def render_GET(self, request):
+        n = getarg(request, "n", 1, type=float)
+        d = deferLater(reactor, n, lambda: (request, n))
+        d.addCallback(self._delayedRender)
+        return NOT_DONE_YET
+
+    def _delayedRender(self, (request, n)):
+        request.write("Response delayed for %0.3f seconds\n" % n)
+        request.finish()
+
 class Log(Resource):
 
     isLeaf = True
@@ -68,6 +83,7 @@ class Root(Resource):
         Resource.__init__(self)
         self.log = []
         self.putChild("follow", Follow())
+        self.putChild("delay", Delay())
         self.putChild("log", Log(self.log))
 
     def getChild(self, request, name):

scrapy/tests/test_crawl.py

@@ -24,6 +24,25 @@ class FollowAllSpider(BaseSpider):
         for link in self.link_extractor.extract_links(response):
             yield Request(link.url, callback=self.parse)
 
+class DelaySpider(BaseSpider):
+
+    name = 'delay'
+
+    def __init__(self, n=1):
+        self.n = n
+        self.t1 = self.t2 = self.t2_err = 0
+
+    def start_requests(self):
+        self.t1 = time.time()
+        yield Request("http://localhost:8998/delay?n=%s" % self.n, \
+            callback=self.parse, errback=self.errback)
+
+    def parse(self, response):
+        self.t2 = time.time()
+
+    def errback(self, failure):
+        self.t2_err = time.time()
+
 def docrawl(spider, settings=None):
     crawler = get_crawler(settings)
     crawler.configure()
@@ -55,3 +74,20 @@ class CrawlTestCase(TestCase):
         for t2 in spider.times[1:]:
             self.assertTrue(t2-t > 0.15, "download delay too small: %s" % (t2-t))
             t = t2
+
+    @defer.inlineCallbacks
+    def test_timeout_success(self):
+        spider = DelaySpider(n=0.5)
+        yield docrawl(spider)
+        self.assertTrue(spider.t1 > 0)
+        self.assertTrue(spider.t2 > 0)
+        self.assertTrue(spider.t2 > spider.t1)
+
+    @defer.inlineCallbacks
+    def test_timeout_failure(self):
+        spider = DelaySpider(n=0.5)
+        yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35})
+        self.assertTrue(spider.t1 > 0)
+        self.assertTrue(spider.t2 == 0)
+        self.assertTrue(spider.t2_err > 0)
+        self.assertTrue(spider.t2_err > spider.t1)