diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index 9f41197c4..fdb6fb462 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -22,6 +22,7 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError from twisted.internet.defer import TimeoutError as UserTimeoutError +from scrapy.xlib.tx._newclient import ResponseFailed from scrapy import log from scrapy.exceptions import NotConfigured @@ -33,7 +34,7 @@ class RetryMiddleware(object): # decompress an empty response EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, - ConnectionLost, TCPTimedOutError, + ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) def __init__(self, settings): diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 4dde6c2a9..1347a7eae 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -102,7 +102,7 @@ class Drop(Partial): def _delayedRender(self, request): request.write("this connection will be dropped\n") - request.channel.transport.loseConnection() + request.channel.transport.abortConnection() request.finish() diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index dcc186c2d..818226ce3 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -90,6 +90,12 @@ class CrawlTestCase(TestCase): yield docrawl(spider) self._assert_retried() + @defer.inlineCallbacks + def test_retry_dropped_connection(self): + spider = SimpleSpider("http://localhost:8998/drop") + yield docrawl(spider) + self._assert_retried() + def _assert_retried(self): log = get_testlog() self.assertEqual(log.count("Retrying"), 2)