test PotentiaDataLoss errors on unbound responses

scrapy/core/downloader/handlers/http11.py

@@ -9,6 +9,7 @@ from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer
 from twisted.internet.error import TimeoutError
+from twisted.web.http import PotentialDataLoss
 from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
     HTTPConnectionPool, TCP4ClientEndpoint
 

scrapy/tests/mockserver.py

@@ -20,6 +20,7 @@ class DeferMixin(object):
             # silence CancelledError
             d.addErrback(lambda _: None)
             d.cancel()
+
         d = deferLater(reactor, delay, f, *a, **kw)
         request.notifyFinish().addErrback(_cancelrequest)
         return d
@@ -84,6 +85,23 @@ class Status(Resource):
         return ""
 
 
+class Raw(DeferMixin, Resource):
+
+    isLeaf = True
+
+    def render_GET(self, request):
+        request.startedWriting = 1
+        self.deferRequest(request, 0, self._delayedRender, request)
+        return NOT_DONE_YET
+
+    def _delayedRender(self, request):
+        raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
+        request.startedWriting = 1
+        request.write(raw)
+        request.channel.transport.loseConnection()
+        request.finish()
+
+
 class Partial(DeferMixin, Resource):
 
     isLeaf = True
@@ -92,6 +110,7 @@ class Partial(DeferMixin, Resource):
         request.setHeader("Content-Length", "1024")
         self.deferRequest(request, 0, self._delayedRender, request)
         return NOT_DONE_YET
+    render_POST = render_GET
 
     def _delayedRender(self, request):
         request.write("partial content\n")
@@ -119,6 +138,7 @@ class Root(Resource):
         self.putChild("delay", Delay())
         self.putChild("partial", Partial())
         self.putChild("drop", Drop())
+        self.putChild("raw", Raw())
 
     def getChild(self, name, request):
         return self

scrapy/tests/test_crawl.py

@@ -90,6 +90,33 @@ class CrawlTestCase(TestCase):
         yield docrawl(spider)
         self._assert_retried()
 
+    @defer.inlineCallbacks
+    def test_unbounded_response(self):
+        # Completeness of responses without Content-Length or Transfer-Encoding
+        # can not be determined, we treat them as valid but flagged as "partial"
+        from urllib import urlencode
+        query = urlencode({'raw': '''\
+HTTP/1.1 200 OK
+Server: Apache-Coyote/1.1
+X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0
+Set-Cookie: JSESSIONID=08515F572832D0E659FD2B0D8031D75F; Path=/
+Pragma: no-cache
+Expires: Thu, 01 Jan 1970 00:00:00 GMT
+Cache-Control: no-cache
+Cache-Control: no-store
+Content-Type: text/html;charset=UTF-8
+Content-Language: en
+Date: Tue, 27 Aug 2013 13:05:05 GMT
+Connection: close
+
+foo body
+with multiples lines
+'''})
+        spider = SimpleSpider("http://localhost:8998/raw?{}".format(query))
+        yield docrawl(spider)
+        log = get_testlog()
+        self.assertEqual(log.count("Got response 200"), 1)
+
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data