HTTP/2: improve header handling

scrapy/core/http2/stream.py

@@ -220,11 +220,24 @@ class Stream:
                 (':path', path),
             ]
 
-        for name, value in self._request.headers.items():
-            headers.append((str(name, 'utf-8'), str(value[0], 'utf-8')))
+        content_length = str(len(self._request.body))
+        headers.append(('Content-Length', content_length))
 
-        if b'Content-Length' not in self._request.headers.keys():
-            headers.append(('Content-Length', str(len(self._request.body))))
+        content_length_name = self._request.headers.normkey(b'Content-Length')
+        for name, values in self._request.headers.items():
+            for value in values:
+                value = str(value, 'utf-8')
+                if name == content_length_name:
+                    if value != content_length:
+                        logger.warning(
+                            'Ignoring bad Content-Length header %r of request %r, '
+                            'sending %r instead',
+                            value,
+                            self._request,
+                            content_length,
+                        )
+                    continue
+                headers.append((str(name, 'utf-8'), value))
 
         return headers
 

tests/test_downloader_handlers_http2.py

@@ -1,6 +1,8 @@
+import json
 from unittest import mock
 
 from pytest import mark
+from testfixtures import LogCapture
 from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
@@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase):
         d.addCallback(self.assertEqual, b'')
         return d
 
+    def test_custom_content_length_good(self):
+        request = Request(self.getURL('contentlength'))
+        custom_content_length = str(len(request.body))
+        request.headers['Content-Length'] = custom_content_length
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.text)
+        d.addCallback(self.assertEqual, custom_content_length)
+        return d
+
+    def test_custom_content_length_bad(self):
+        request = Request(self.getURL('contentlength'))
+        actual_content_length = str(len(request.body))
+        bad_content_length = str(len(request.body)+1)
+        request.headers['Content-Length'] = bad_content_length
+        log = LogCapture()
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.text)
+        d.addCallback(self.assertEqual, actual_content_length)
+        d.addCallback(
+            lambda _: log.check_present(
+                (
+                    'scrapy.core.http2.stream',
+                    'WARNING',
+                    f'Ignoring bad Content-Length header '
+                    f'{bad_content_length!r} of request {request}, sending '
+                    f'{actual_content_length!r} instead',
+                )
+            )
+        )
+        d.addCallback(
+            lambda _: log.uninstall()
+        )
+        return d
+
+    def test_duplicate_header(self):
+        request = Request(self.getURL('echo'))
+        header, value1, value2 = 'Custom-Header', 'foo', 'bar'
+        request.headers.appendlist(header, value1)
+        request.headers.appendlist(header, value2)
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: json.loads(r.text)['headers'][header])
+        d.addCallback(self.assertEqual, [value1, value2])
+        return d
+
 
 class Https2WrongHostnameTestCase(Https2TestCase):
     tls_log_message = (