diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index fdb6fb462..7eba3f78d 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -18,15 +18,16 @@ About HTTP errors to consider: indicate server overload, which would be something we want to retry """ -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError from twisted.internet.defer import TimeoutError as UserTimeoutError -from scrapy.xlib.tx._newclient import ResponseFailed +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost, TCPTimedOutError from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message +from scrapy.xlib.tx import ResponseFailed + class RetryMiddleware(object): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6ec10f723..c74581991 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,17 +7,16 @@ from urlparse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer +from twisted.web.http import PotentialDataLoss from twisted.internet.error import TimeoutError from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint + HTTPConnectionPool, TCP4ClientEndpoint, ResponseFailed from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object -from scrapy import log class HTTP11DownloadHandler(object): @@ -55,7 +54,7 @@ class ScrapyAgent(object): if proxy: scheme, _, host, port, _ = _parse(proxy) endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, - bindAddress=bindaddress) + bindAddress=bindaddress) return self._ProxyAgent(endpoint) return self._Agent(reactor, contextFactory=self._contextFactory, @@ -145,10 +144,9 @@ class _ResponseReader(protocol.Protocol): def connectionLost(self, reason): if self._finished.called: return + body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss, ResponseFailed): - self._finished.callback((self._txresponse, body, ['partial'])) else: self._finished.errback(reason) diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 1347a7eae..e91fd1194 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -101,9 +101,13 @@ class Partial(DeferMixin, Resource): class Drop(Partial): def _delayedRender(self, request): + abort = getarg(request, "abort", 0, type=int) request.write("this connection will be dropped\n") - request.channel.transport.abortConnection() - request.finish() + if abort: + request.channel.transport.abortConnection() + else: + request.channel.transport.loseConnection() + request.finish() class Root(Resource): diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 818226ce3..7e5a80512 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -91,8 +91,16 @@ class CrawlTestCase(TestCase): self._assert_retried() @defer.inlineCallbacks - def test_retry_dropped_connection(self): - spider = SimpleSpider("http://localhost:8998/drop") + def test_retry_conn_lost(self): + # connection lost after receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=0") + yield docrawl(spider) + self._assert_retried() + + @defer.inlineCallbacks + def test_retry_conn_aborted(self): + # connection lost before receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=1") yield docrawl(spider) self._assert_retried() diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index d9bf1a543..c398afdba 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -1,14 +1,16 @@ import unittest -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware +from scrapy.xlib.tx import ResponseFailed from scrapy.spider import BaseSpider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler + class RetryTest(unittest.TestCase): def setUp(self): crawler = get_crawler() @@ -62,9 +64,11 @@ class RetryTest(unittest.TestCase): assert self.mw.process_response(req, rsp, self.spider) is rsp def test_twistederrors(self): - for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost): + for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, + ConnectionDone, ConnectError, ConnectionLost, + ResponseFailed): req = Request('http://www.scrapytest.org/%s' % exc.__name__) - self._test_retry_exception(req, exc()) + self._test_retry_exception(req, exc('foo')) def _test_retry_exception(self, req, exception): # first retry