fix crawling tests under twisted pre 11.0.0

scrapy/core/downloader/webclient.py

@@ -58,12 +58,15 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.factory.gotHeaders(self.headers)
 
     def connectionLost(self, reason):
+        self._connection_lost_reason = reason
         HTTPClient.connectionLost(self, reason)
         self.factory.noPage(reason)
 
     def handleResponse(self, response):
         if self.factory.method.upper() == 'HEAD':
             self.factory.page('')
+        elif self.length is not None and self.length > 0:
+            self.factory.noPage(self._connection_lost_reason)
         else:
             self.factory.page(response)
         self.transport.loseConnection()

scrapy/tests/mockserver.py

@@ -2,8 +2,29 @@ import sys, time, random, urllib
 from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
-from twisted.internet import reactor
-from twisted.internet.task import deferLater
+from twisted.internet import reactor, defer
+from scrapy import twisted_version
+
+
+if twisted_version < (11, 0, 0):
+    def deferLater(clock, delay, func, *args, **kw):
+        def _cancel_method():
+            _cancel_cb(None)
+            d.errback(Exception())
+
+        def _cancel_cb(result):
+            if cl.active():
+                cl.cancel()
+            return result
+
+        d = defer.Deferred()
+        d.cancel = _cancel_method
+        d.addCallback(lambda ignored: func(*args, **kw))
+        d.addBoth(_cancel_cb)
+        cl = clock.callLater(delay, d.callback, None)
+        return d
+else:
+    from twisted.internet.task import deferLater
 
 
 def getarg(request, name, default=None, type=str):
@@ -13,7 +34,9 @@ def getarg(request, name, default=None, type=str):
         return default
 
 
-class DeferMixin(object):
+class LeafResource(Resource):
+
+    isLeaf = True
 
     def deferRequest(self, request, delay, f, *a, **kw):
         def _cancelrequest(_):
@@ -26,9 +49,7 @@ class DeferMixin(object):
         return d
 
 
-class Follow(DeferMixin, Resource):
-
-    isLeaf = True
+class Follow(LeafResource):
 
     def render(self, request):
         total = getarg(request, "total", 100, type=int)
@@ -57,9 +78,7 @@ class Follow(DeferMixin, Resource):
         request.finish()
 
 
-class Delay(DeferMixin, Resource):
-
-    isLeaf = True
+class Delay(LeafResource):
 
     def render_GET(self, request):
         n = getarg(request, "n", 1, type=float)
@@ -75,9 +94,7 @@ class Delay(DeferMixin, Resource):
         request.finish()
 
 
-class Status(Resource):
-
-    isLeaf = True
+class Status(LeafResource):
 
     def render_GET(self, request):
         n = getarg(request, "n", 200, type=int)
@@ -85,14 +102,13 @@ class Status(Resource):
         return ""
 
 
-class Raw(DeferMixin, Resource):
-
-    isLeaf = True
+class Raw(LeafResource):
 
     def render_GET(self, request):
         request.startedWriting = 1
         self.deferRequest(request, 0, self._delayedRender, request)
         return NOT_DONE_YET
+    render_POST = render_GET
 
     def _delayedRender(self, request):
         raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
@@ -102,15 +118,12 @@ class Raw(DeferMixin, Resource):
         request.finish()
 
 
-class Partial(DeferMixin, Resource):
-
-    isLeaf = True
+class Partial(LeafResource):
 
     def render_GET(self, request):
         request.setHeader("Content-Length", "1024")
         self.deferRequest(request, 0, self._delayedRender, request)
         return NOT_DONE_YET
-    render_POST = render_GET
 
     def _delayedRender(self, request):
         request.write("partial content\n")
@@ -122,10 +135,13 @@ class Drop(Partial):
     def _delayedRender(self, request):
         abort = getarg(request, "abort", 0, type=int)
         request.write("this connection will be dropped\n")
-        if abort:
-            request.channel.transport.abortConnection()
-        else:
-            request.channel.transport.loseConnection()
+        tr = request.channel.transport
+        try:
+            if abort and hasattr(tr, 'abortConnection'):
+                tr.abortConnection()
+            else:
+                tr.loseConnection()
+        finally:
             request.finish()
 
 
@@ -165,6 +181,7 @@ if __name__ == "__main__":
     root = Root()
     factory = Site(root)
     port = reactor.listenTCP(8998, factory)
+
     def print_listening():
         h = port.getHost()
         print "Mock server running at http://%s:%d" % (h.host, h.port)