From 1b6d5a011a5485f3f2c01b32fa425b512d90d7ef Mon Sep 17 00:00:00 2001 From: drack3800 Date: Sat, 21 Mar 2015 04:02:51 +0300 Subject: [PATCH 1/3] Added webclient test for checking Content-Length header in response for POST request with no given body --- tests/test_webclient.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a16eb1ccf..e0b46286a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -132,6 +132,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): "\r\n" "name=value") + # test a POST method with no body provided + factory = client.ScrapyHTTPClientFactory(Request( + method='POST', + url='http://foo/bar' + )) + + self._test(factory, + "POST /bar HTTP/1.0\r\n" + "Host: foo\r\n" + "Content-Length: 0\r\n" + "\r\n") + # test with single and multivalued headers factory = client.ScrapyHTTPClientFactory(Request( url='http://foo/bar', From deb5bb530cfe14993f92d3031820153ae53e0edb Mon Sep 17 00:00:00 2001 From: drack3800 Date: Sun, 22 Mar 2015 19:25:08 +0300 Subject: [PATCH 2/3] Fixed bug with no specified Content-Length header by ScrapyHTTPClientFactory for POST request with no given body --- scrapy/core/downloader/webclient.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 2c6a61b8a..93ab8a391 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -112,12 +112,14 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # set Host header based on url self.headers.setdefault('Host', self.netloc) - # set Content-Length based len of body if self.body is not None: self.headers['Content-Length'] = len(self.body) # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") + # Content-Length must be specified in POST method even with no body + elif self.method == 'POST': + self.headers['Content-Length'] = 0 def _build_response(self, body, request): request.meta['download_latency'] = self.headers_time-self.start_time From 549882590bfdb3d66b623ea7414fac5097560a96 Mon Sep 17 00:00:00 2001 From: drack3800 Date: Sun, 22 Mar 2015 19:25:08 +0300 Subject: [PATCH 3/3] Fixed bug with no specified Content-Length header by ScrapyHTTPClientFactory for POST request with no given body --- scrapy/core/downloader/webclient.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 2c6a61b8a..22c461b15 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -112,12 +112,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # set Host header based on url self.headers.setdefault('Host', self.netloc) - + # set Content-Length based len of body if self.body is not None: self.headers['Content-Length'] = len(self.body) # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") + # Content-Length must be specified in POST method even with no body + elif self.method == 'POST': + self.headers['Content-Length'] = 0 def _build_response(self, body, request): request.meta['download_latency'] = self.headers_time-self.start_time