Fixed issue with non-standard line ending in HTTP headers. Closes #258

scrapy/core/downloader/webclient.py

@@ -33,6 +33,8 @@ def _parse(url):
 
 class ScrapyHTTPPageGetter(HTTPClient):
 
+    delimiter = '\n'
+
     def connectionMade(self):
         self.headers = Headers() # bucket for response headers
 
@@ -47,6 +49,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
         if self.factory.body is not None:
             self.transport.write(self.factory.body)
 
+    def lineReceived(self, line):
+        return HTTPClient.lineReceived(self, line.rstrip())
+
     def handleHeader(self, key, value):
         self.headers.appendlist(key, value)
 

scrapy/tests/test_webclient.py

@@ -172,6 +172,20 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         self.assertEqual(transport.value(), testvalue)
         return testvalue
 
+    def test_non_standard_line_endings(self):
+        # regression test for: http://dev.scrapy.org/ticket/258
+        factory = client.ScrapyHTTPClientFactory(Request(
+            url='http://foo/bar'))
+        protocol = client.ScrapyHTTPPageGetter()
+        protocol.factory = factory
+        protocol.headers = Headers()
+        protocol.dataReceived("HTTP/1.0 200 OK\n")
+        protocol.dataReceived("Hello: World\n")
+        protocol.dataReceived("Foo: Bar\n")
+        protocol.dataReceived("\n")
+        self.assertEqual(protocol.headers,
+            Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+
 
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         ErrorResource, NoLengthResource, HostHeaderResource, \