diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5017f9cd4..a2f0e2aa4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,7 +16,6 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -177,8 +176,8 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == to_unicode(self._protocol.metadata['uri'].host) - or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') + or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) @@ -394,7 +393,7 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - to_unicode(self._protocol.metadata['uri'].host), + str(self._protocol.metadata['uri'].host, 'utf-8'), f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05e5f5047..05f4889ef 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,7 +23,6 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -39,7 +38,7 @@ def make_html_body(val):

Hello from HTTP2

{val}

''' - return to_bytes(response) + return bytes(response, 'utf-8') class Data: @@ -83,10 +82,11 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][to_unicode(k)] = to_unicode(v[0]) + response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') - response_bytes = to_bytes(json.dumps(response)) - request.setHeader('Content-Type', 'application/json') + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') return response_bytes @@ -127,13 +127,14 @@ class NoContentLengthHeader(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') query_params = {} for k, v in request.args.items(): - query_params[to_unicode(k)] = to_unicode(v[0]) + query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') - return to_bytes(json.dumps(query_params)) + return bytes(json.dumps(query_params), 'utf-8') def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @@ -184,7 +185,7 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - uri = URI.fromBytes(to_bytes(self.get_url('/'))) + uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -278,7 +279,8 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + body = json.loads(str(response.body, content_encoding)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -292,9 +294,9 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = to_unicode(k) + k_str = str(k, 'utf-8') self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], to_unicode(v[0])) + self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) d.addCallback(assert_response) d.addErrback(self.fail) @@ -494,7 +496,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f'/query-params?{urlencode(params)}')) def assert_query_params(response: Response): - data = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) d = self.client.request(request)