Do not raise PartialDownloadError if Content-Length doesn't match the body size. This fixes the error reported in: https://groups.google.com/d/topic/scrapy-users/FQ25O3KPQuU/discussion

scrapy/contrib/downloadermiddleware/retry.py

@@ -22,7 +22,6 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup
                                    ConnectionRefusedError, ConnectionDone, ConnectError, \
                                    ConnectionLost, TCPTimedOutError
 from twisted.internet.defer import TimeoutError as UserTimeoutError
-from twisted.web.client import PartialDownloadError
 
 from scrapy import log
 from scrapy.exceptions import NotConfigured
@@ -35,7 +34,7 @@ class RetryMiddleware(object):
     # decompress an empty response
     EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, PartialDownloadError, TCPTimedOutError,
+                           ConnectionLost, TCPTimedOutError,
                            IOError)
 
     def __init__(self):

scrapy/core/downloader/webclient.py

@@ -2,7 +2,7 @@ from time import time
 from urlparse import urlparse, urlunparse, urldefrag
 
 from twisted.python import failure
-from twisted.web.client import PartialDownloadError, HTTPClientFactory
+from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
 
@@ -65,9 +65,6 @@ class ScrapyHTTPPageGetter(HTTPClient):
     def handleResponse(self, response):
         if self.factory.method.upper() == 'HEAD':
             self.factory.page('')
-        elif self.length != None and self.length != 0:
-            self.factory.noPage(failure.Failure(
-                PartialDownloadError(self.factory.status, None, response)))
         else:
             self.factory.page(response)
         self.transport.loseConnection()

scrapy/tests/test_downloader_handlers.py

@@ -10,7 +10,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
         PayloadResource, BrokenDownloadResource
 from w3lib.url import path_to_file_uri
 
-from scrapy.core.downloader.webclient import PartialDownloadError
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HttpDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
@@ -131,11 +130,6 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEquals, body)
         return d
 
-    def test_broken_download(self):
-        request = Request(self.getURL('broken'))
-        d = self.download_request(request, BaseSpider('foo'))
-        return self.assertFailure(d, PartialDownloadError)
-
 
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""

scrapy/tests/test_webclient.py

@@ -222,13 +222,6 @@ class WebClientTestCase(unittest.TestCase):
         s = "0123456789" * 10
         return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
 
-    def testBrokenDownload(self):
-        # test what happens when download gets disconnected in the middle
-        d = getPage(self.getURL("broken"))
-        d = self.assertFailure(d, client.PartialDownloadError)
-        d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
-        return d
-
     def testHostHeader(self):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url