From f4821a123d17e90686ea0b1eb9447dedcb604431 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 25 Oct 2011 13:04:58 -0200
Subject: [PATCH] Do not raise PartialDownloadError if Content-Length doesn't
 match the body size. This fixes the error reported in:
 https://groups.google.com/d/topic/scrapy-users/FQ25O3KPQuU/discussion

---
 scrapy/contrib/downloadermiddleware/retry.py | 3 +--
 scrapy/core/downloader/webclient.py          | 5 +----
 scrapy/tests/test_downloader_handlers.py     | 6 ------
 scrapy/tests/test_webclient.py               | 7 -------
 4 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
index 71fb3a94f..807c30622 100644
--- a/scrapy/contrib/downloadermiddleware/retry.py
+++ b/scrapy/contrib/downloadermiddleware/retry.py
@@ -22,7 +22,6 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup
                                    ConnectionRefusedError, ConnectionDone, ConnectError, \
                                    ConnectionLost, TCPTimedOutError
 from twisted.internet.defer import TimeoutError as UserTimeoutError
-from twisted.web.client import PartialDownloadError
 
 from scrapy import log
 from scrapy.exceptions import NotConfigured
@@ -35,7 +34,7 @@ class RetryMiddleware(object):
     # decompress an empty response
     EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, PartialDownloadError, TCPTimedOutError,
+                           ConnectionLost, TCPTimedOutError,
                            IOError)
 
     def __init__(self):
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index a1eaa7a07..018e1d050 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -2,7 +2,7 @@ from time import time
 from urlparse import urlparse, urlunparse, urldefrag
 
 from twisted.python import failure
-from twisted.web.client import PartialDownloadError, HTTPClientFactory
+from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
 
@@ -65,9 +65,6 @@ class ScrapyHTTPPageGetter(HTTPClient):
     def handleResponse(self, response):
         if self.factory.method.upper() == 'HEAD':
             self.factory.page('')
-        elif self.length != None and self.length != 0:
-            self.factory.noPage(failure.Failure(
-                PartialDownloadError(self.factory.status, None, response)))
         else:
             self.factory.page(response)
         self.transport.loseConnection()
diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py
index 31d024802..ae1fb3460 100644
--- a/scrapy/tests/test_downloader_handlers.py
+++ b/scrapy/tests/test_downloader_handlers.py
@@ -10,7 +10,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
         PayloadResource, BrokenDownloadResource
 from w3lib.url import path_to_file_uri
 
-from scrapy.core.downloader.webclient import PartialDownloadError
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HttpDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
@@ -131,11 +130,6 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEquals, body)
         return d
 
-    def test_broken_download(self):
-        request = Request(self.getURL('broken'))
-        d = self.download_request(request, BaseSpider('foo'))
-        return self.assertFailure(d, PartialDownloadError)
-
 
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""
diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py
index 60ec83b40..9de09f343 100644
--- a/scrapy/tests/test_webclient.py
+++ b/scrapy/tests/test_webclient.py
@@ -222,13 +222,6 @@ class WebClientTestCase(unittest.TestCase):
         s = "0123456789" * 10
         return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
 
-    def testBrokenDownload(self):
-        # test what happens when download gets disconnected in the middle
-        d = getPage(self.getURL("broken"))
-        d = self.assertFailure(d, client.PartialDownloadError)
-        d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
-        return d
-
     def testHostHeader(self):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url