From 0b91c04007c8875f7d9a29cd5e151a91939c5160 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Sat, 9 Oct 2010 20:43:05 -0200
Subject: [PATCH] Fixed issue with non-standard line ending in HTTP headers.
 Closes #258

---
 scrapy/core/downloader/webclient.py |  5 +++++
 scrapy/tests/test_webclient.py      | 14 ++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 0aeb861e0..713bf6aab 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -33,6 +33,8 @@ def _parse(url):
 
 class ScrapyHTTPPageGetter(HTTPClient):
 
+    delimiter = '\n'
+
     def connectionMade(self):
         self.headers = Headers() # bucket for response headers
 
@@ -47,6 +49,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
         if self.factory.body is not None:
             self.transport.write(self.factory.body)
 
+    def lineReceived(self, line):
+        return HTTPClient.lineReceived(self, line.rstrip())
+
     def handleHeader(self, key, value):
         self.headers.appendlist(key, value)
 
diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py
index 260b2bc08..60ec83b40 100644
--- a/scrapy/tests/test_webclient.py
+++ b/scrapy/tests/test_webclient.py
@@ -172,6 +172,20 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         self.assertEqual(transport.value(), testvalue)
         return testvalue
 
+    def test_non_standard_line_endings(self):
+        # regression test for: http://dev.scrapy.org/ticket/258
+        factory = client.ScrapyHTTPClientFactory(Request(
+            url='http://foo/bar'))
+        protocol = client.ScrapyHTTPPageGetter()
+        protocol.factory = factory
+        protocol.headers = Headers()
+        protocol.dataReceived("HTTP/1.0 200 OK\n")
+        protocol.dataReceived("Hello: World\n")
+        protocol.dataReceived("Foo: Bar\n")
+        protocol.dataReceived("\n")
+        self.assertEqual(protocol.headers,
+            Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+
 
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         ErrorResource, NoLengthResource, HostHeaderResource, \