diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0f3730c9e..0a2719d23 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,26 +1,26 @@ -# TODO: Add test cases for -# 1. No Content Length response header -# 2. Cancel Response Deferred import json import os import random +import re import shutil import string +from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest -from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.http import Request, Response, JsonRequest -from tests.mockserver import ssl_context_factory +from scrapy.utils.python import to_bytes, to_unicode +from tests.mockserver import ssl_context_factory, LeafResource def generate_random_string(size): @@ -35,7 +35,7 @@ def make_html_body(val):

Hello from HTTP2

{}

'''.format(val) - return bytes(response, 'utf-8') + return to_bytes(response) class Data: @@ -54,9 +54,8 @@ class Data: JSON_SMALL = {'data': STR_SMALL} JSON_LARGE = {'data': STR_LARGE} - -class LeafResource(Resource): - isLeaf = True + DATALOSS = b'Dataloss Content' + NO_CONTENT_LENGTH = b'This response do not have any content-length header' class GetDataHtmlSmall(LeafResource): @@ -80,9 +79,9 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + response['request-headers'][to_unicode(k)] = to_unicode(v[0]) - response_bytes = bytes(json.dumps(response), 'utf-8') + response_bytes = to_bytes(json.dumps(response)) request.setHeader('Content-Type', 'application/json') return response_bytes @@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): return self.make_response(request, Data.EXTRA_LARGE) +class Dataloss(LeafResource): + + def render_GET(self, request: TxRequest): + request.setHeader(b"Content-Length", b"1024") + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.DATALOSS) + request.finish() + + +class NoContentLengthHeader(LeafResource): + def render_GET(self, request: TxRequest): + request.requestHeaders.removeHeader('Content-Length') + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.NO_CONTENT_LENGTH) + request.finish() + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'post-data-json-small', PostDataJsonSmall()) r.putChild(b'post-data-json-large', PostDataJsonLarge()) + + r.putChild(b'dataloss', Dataloss()) + r.putChild(b'no-content-length-header', NoContentLengthHeader()) return r @inlineCallbacks @@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def _check_GET( - self, - request: Request, - expected_body, - expected_status + self, + request: Request, + expected_body, + expected_status ): def check_response(response: Response): self.assertEqual(response.status, expected_status) @@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase): ) def _check_POST_json( - self, - request: Request, - expected_request_body, - expected_extra_data, - expected_status: int + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int ): d = self.client.request(request) @@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(response.body.decode('utf-8')) + body = json.loads(to_unicode(response.body)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = k.decode('utf-8') + k_str = to_unicode(k) self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + self.assertEqual(request_headers[k_str], to_unicode(v[0])) d.addCallback(assert_response) + d.addErrback(self.fail) return d def test_POST_small_json(self): @@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase): d = self.client.request(request) d.addCallback(assert_response) + d.addErrback(self.fail) d.cancel() return d + + def test_download_maxsize_exceeded(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + + def assert_cancelled_error(failure): + self.assertIsInstance(failure.value, CancelledError) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_cancelled_error) + return d + + # TODO: Test in multiple requests if one request fails due to dataloss + # remaining request do not fail (change expected behaviour) + # Can be done only when hyper-h2 don't terminate connection over + # InvalidBodyLengthError check + def test_received_dataloss_response(self): + """In case when value of Header Content-Length != len(Received Data) + ProtocolError is raised""" + request = Request(url=self.get_url('/dataloss')) + + def assert_failure(failure: Failure): + self.assertTrue(len(failure.value.reasons) > 0) + self.assertTrue(any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + )) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_failure) + return d + + def test_missing_content_length_header(self): + request = Request(url=self.get_url('/no-content-length-header')) + + def assert_content_length(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertIn('partial', response.flags) + self.assertNotIn('Content-Length', response.headers) + + d = self.client.request(request) + d.addCallback(assert_content_length) + d.addErrback(self.fail) + return d + + @inlineCallbacks + def _check_log_warnsize( + self, + request, + warn_pattern, + expected_body + ): + with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + response = yield self.client.request(request) + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, expected_body) + + # Check the warning is raised only once for this request + self.assertEqual(sum( + len(re.findall(warn_pattern, log)) + for log in cm.output + ), 1) + + @inlineCallbacks + def test_log_expected_warnsize(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + warn_pattern = re.compile( + r'Expected response size \(\d*\) larger than ' + r'download warn size \(1000\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + + @inlineCallbacks + def test_log_received_warnsize(self): + request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + warn_pattern = re.compile( + r'Received more \(\d*\) bytes than download ' + r'warn size \(10\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)