diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index aa44c08ce..8a701e7c6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -220,11 +220,24 @@ class Stream: (':path', path), ] - for name, value in self._request.headers.items(): - headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + content_length = str(len(self._request.body)) + headers.append(('Content-Length', content_length)) - if b'Content-Length' not in self._request.headers.keys(): - headers.append(('Content-Length', str(len(self._request.body)))) + content_length_name = self._request.headers.normkey(b'Content-Length') + for name, values in self._request.headers.items(): + for value in values: + value = str(value, 'utf-8') + if name == content_length_name: + if value != content_length: + logger.warning( + 'Ignoring bad Content-Length header %r of request %r, ' + 'sending %r instead', + value, + self._request, + content_length, + ) + continue + headers.append((str(name, 'utf-8'), value)) return headers diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index b5a40468a..7c3db5835 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,6 +1,8 @@ +import json from unittest import mock from pytest import mark +from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase): d.addCallback(self.assertEqual, b'') return d + def test_custom_content_length_good(self): + request = Request(self.getURL('contentlength')) + custom_content_length = str(len(request.body)) + request.headers['Content-Length'] = custom_content_length + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, custom_content_length) + return d + + def test_custom_content_length_bad(self): + request = Request(self.getURL('contentlength')) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body)+1) + request.headers['Content-Length'] = bad_content_length + log = LogCapture() + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, actual_content_length) + d.addCallback( + lambda _: log.check_present( + ( + 'scrapy.core.http2.stream', + 'WARNING', + f'Ignoring bad Content-Length header ' + f'{bad_content_length!r} of request {request}, sending ' + f'{actual_content_length!r} instead', + ) + ) + ) + d.addCallback( + lambda _: log.uninstall() + ) + return d + + def test_duplicate_header(self): + request = Request(self.getURL('echo')) + header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request.headers.appendlist(header, value1) + request.headers.appendlist(header, value2) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d.addCallback(self.assertEqual, [value1, value2]) + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = (