fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373

scrapy/contrib/downloadermiddleware/retry.py

@@ -22,6 +22,7 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup
                                    ConnectionRefusedError, ConnectionDone, ConnectError, \
                                    ConnectionLost, TCPTimedOutError
 from twisted.internet.defer import TimeoutError as UserTimeoutError
+from scrapy.xlib.tx._newclient import ResponseFailed
 
 from scrapy import log
 from scrapy.exceptions import NotConfigured
@@ -33,7 +34,7 @@ class RetryMiddleware(object):
     # decompress an empty response
     EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, TCPTimedOutError,
+                           ConnectionLost, TCPTimedOutError, ResponseFailed,
                            IOError)
 
     def __init__(self, settings):

scrapy/tests/mockserver.py

@@ -102,7 +102,7 @@ class Drop(Partial):
 
     def _delayedRender(self, request):
         request.write("this connection will be dropped\n")
-        request.channel.transport.loseConnection()
+        request.channel.transport.abortConnection()
         request.finish()
 
 

scrapy/tests/test_crawl.py

@@ -90,6 +90,12 @@ class CrawlTestCase(TestCase):
         yield docrawl(spider)
         self._assert_retried()
 
+    @defer.inlineCallbacks
+    def test_retry_dropped_connection(self):
+        spider = SimpleSpider("http://localhost:8998/drop")
+        yield docrawl(spider)
+        self._assert_retried()
+
     def _assert_retried(self):
         log = get_testlog()
         self.assertEqual(log.count("Retrying"), 2)