From e25052e4ef62ae5b3ef662bb29da13930c7b6dba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 20 Feb 2016 23:27:04 +0100 Subject: [PATCH] Add "Content-Length: 0" for body-less HTTP/1.1 POST requests GH-823 was fixed only for HTTP/1.0 (in GH-1089) --- scrapy/core/downloader/handlers/http11.py | 16 +++++++++++- tests/test_downloader_handlers.py | 32 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index ad3285a32..ca13c609d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -209,7 +209,21 @@ class ScrapyAgent(object): headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b'Proxy-Authorization') - bodyproducer = _RequestBodyProducer(request.body) if request.body else None + if request.body: + bodyproducer = _RequestBodyProducer(request.body) + else: + bodyproducer = None + # Setting Content-Length: 0 even for POST requests is not a + # MUST per HTTP RFCs, but it's common behavior, and some + # server require this otherwise returing HTTP 411 + # + # RFC 7230#section-3.3.2: + # "a Content-Length header field is normally sent in a POST + # request even when the value is 0 (indicating an empty payload body)." + # + # Twisted Agent will not add "Content-Length: 0" by itself + if method == b'POST': + headers.addRawHeader(b'Content-Length', b'0') start_time = time() d = agent.request( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1885a53a0..88b60e4ff 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -105,6 +105,15 @@ class FileTestCase(unittest.TestCase): return self.assertFailure(d, IOError) +class ContentLengthHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the Content-Length + header from the request. + """ + def render(self, request): + return request.requestHeaders.getRawHeaders(b"content-length")[0] + + class HttpTestCase(unittest.TestCase): scheme = 'http' @@ -122,6 +131,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"contentlength", ContentLengthHeaderResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' @@ -209,6 +219,28 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEquals, b'example.com') return d + def test_content_length_zero_bodyless_post_request_headers(self): + """Tests if "Content-Length: 0" is sent for bodyless POST requests. + + This is not strictly required by HTTP RFCs but can cause trouble + for some web servers. + See: + https://github.com/scrapy/scrapy/issues/823 + https://issues.apache.org/jira/browse/TS-2902 + https://github.com/kennethreitz/requests/issues/405 + https://bugs.python.org/issue14721 + """ + def _test(response): + self.assertEquals(response.body, b'0') + + request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) + return self.download_request(request, Spider('foo')).addCallback(_test) + + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, b'0') + return d + def test_payload(self): body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body)