New HTTP client wraps connection losts in ResponseFailed exception. fix #373

scrapy/contrib/downloadermiddleware/retry.py

@@ -18,15 +18,16 @@ About HTTP errors to consider:
   indicate server overload, which would be something we want to retry
 """
 
-from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
-                                   ConnectionRefusedError, ConnectionDone, ConnectError, \
-                                   ConnectionLost, TCPTimedOutError
 from twisted.internet.defer import TimeoutError as UserTimeoutError
-from scrapy.xlib.tx._newclient import ResponseFailed
+from twisted.internet.error import TimeoutError as ServerTimeoutError, \
+        DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \
+        ConnectionLost, TCPTimedOutError
 
 from scrapy import log
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
+from scrapy.xlib.tx import ResponseFailed
+
 
 class RetryMiddleware(object):
 

scrapy/core/downloader/handlers/http11.py

@@ -7,17 +7,16 @@ from urlparse import urldefrag
 from zope.interface import implements
 from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.http import PotentialDataLoss
 from twisted.web.iweb import IBodyProducer
+from twisted.web.http import PotentialDataLoss
 from twisted.internet.error import TimeoutError
 from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
-        ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint
+    HTTPConnectionPool, TCP4ClientEndpoint, ResponseFailed
 
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
-from scrapy import log
 
 
 class HTTP11DownloadHandler(object):
@@ -55,7 +54,7 @@ class ScrapyAgent(object):
         if proxy:
             scheme, _, host, port, _ = _parse(proxy)
             endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout,
-                bindAddress=bindaddress)
+                                          bindAddress=bindaddress)
             return self._ProxyAgent(endpoint)
 
         return self._Agent(reactor, contextFactory=self._contextFactory,
@@ -145,10 +144,9 @@ class _ResponseReader(protocol.Protocol):
     def connectionLost(self, reason):
         if self._finished.called:
             return
+
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
             self._finished.callback((self._txresponse, body, None))
-        elif reason.check(PotentialDataLoss, ResponseFailed):
-            self._finished.callback((self._txresponse, body, ['partial']))
         else:
             self._finished.errback(reason)

scrapy/tests/mockserver.py

@@ -101,9 +101,13 @@ class Partial(DeferMixin, Resource):
 class Drop(Partial):
 
     def _delayedRender(self, request):
+        abort = getarg(request, "abort", 0, type=int)
         request.write("this connection will be dropped\n")
-        request.channel.transport.abortConnection()
+        if abort:
-        request.finish()
+            request.channel.transport.abortConnection()
+        else:
+            request.channel.transport.loseConnection()
+            request.finish()
 
 
 class Root(Resource):

scrapy/tests/test_crawl.py

@@ -91,8 +91,16 @@ class CrawlTestCase(TestCase):
         self._assert_retried()
 
     @defer.inlineCallbacks
-    def test_retry_dropped_connection(self):
+    def test_retry_conn_lost(self):
-        spider = SimpleSpider("http://localhost:8998/drop")
+        # connection lost after receiving data
+        spider = SimpleSpider("http://localhost:8998/drop?abort=0")
+        yield docrawl(spider)
+        self._assert_retried()
+
+    @defer.inlineCallbacks
+    def test_retry_conn_aborted(self):
+        # connection lost before receiving data
+        spider = SimpleSpider("http://localhost:8998/drop?abort=1")
         yield docrawl(spider)
         self._assert_retried()
 

scrapy/tests/test_downloadermiddleware_retry.py

@@ -1,14 +1,16 @@
 import unittest
 
-from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
+from twisted.internet.error import TimeoutError as ServerTimeoutError, \
-                                   ConnectionRefusedError, ConnectionDone, ConnectError, \
+    DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \
-                                   ConnectionLost
+    ConnectionLost
 
 from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
+from scrapy.xlib.tx import ResponseFailed
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
 from scrapy.utils.test import get_crawler
 
+
 class RetryTest(unittest.TestCase):
     def setUp(self):
         crawler = get_crawler()
@@ -62,9 +64,11 @@ class RetryTest(unittest.TestCase):
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_twistederrors(self):
-        for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost):
+        for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError,
+                    ConnectionDone, ConnectError, ConnectionLost,
+                    ResponseFailed):
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)
-            self._test_retry_exception(req, exc())
+            self._test_retry_exception(req, exc('foo'))
 
     def _test_retry_exception(self, req, exception):
         # first retry