From a97ac0adf86b67a16f09c41f618f736adb872503 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 06:40:20 +0530 Subject: [PATCH] test: GET request for HTTP2Client using mockserver --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 10 +--- tests/test_http2_client_protocol.py | 71 ++++++++++------------------- 3 files changed, 30 insertions(+), 55 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 188c14c15..455d8777e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,6 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) + +from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol from scrapy.core.http2.stream import Stream, StreamCloseReason @@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol): self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) - self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 07a4428c8..112ce5bcd 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -117,7 +117,7 @@ class Stream: self._deferred_response = Deferred() def __str__(self): - return "Stream(id={})".format(self.stream_id) + return "Stream(id={})".format(repr(self.stream_id)) __repr__ = __str__ @@ -299,9 +299,6 @@ class Stream: def close(self, reason: StreamCloseReason): """Based on the reason sent we will handle each case. """ - # TODO: In case of abruptly stream close - # Q1. Do we need to send the request again? - # Q2. What response should we send now? if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -346,10 +343,7 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 2. Should we fire this in case of - # 2.1 StreamReset in between when data is received partially - # 2.2 Forcefully closed the stream - # 3. Update Client Side Status Codes here + # 1. Update Client Side Status Codes here response_cls = responsetypes.from_args( headers=self._response['headers'], diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7830f7028..a67575d3c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,71 +1,50 @@ -import os -import shutil +from urllib.parse import urlparse -from twisted.internet import defer, reactor +from twisted.internet import reactor from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS -from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath +from twisted.internet.ssl import CertificateOptions from twisted.trial import unittest -from twisted.web import static, server from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request -from tests.mockserver import ssl_context_factory +from scrapy.http import Request, Response +from tests.mockserver import MockServer class Http2ClientProtocolTestCase(unittest.TestCase): scheme = 'https' - # only used for HTTPS tests - file_key = 'keys/localhost.key' - file_certificate = 'keys/localhost.crt' - def setUp(self): # Start server for testing - self.path_temp = self.mktemp() - os.mkdir(self.path_temp) - FilePath(self.path_temp).child('file').setContent(b"0123456789") - r = static.File(self.path_temp) + self.mockserver = MockServer() + self.mockserver.__enter__() - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' - if self.scheme is 'https': - self.port = reactor.listenSSL( - 0, self.wrapper, - ssl_context_factory(self.file_key, self.file_certificate), - interface=self.host - ) + if self.scheme == 'https': + self.url = urlparse(self.mockserver.https_address) else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - - self.port_number = self.port.getHost().port - - # Connect to the server using the custom HTTP2ClientProtocol - options = optionsForClientTLS( - hostname=self.host, - acceptableProtocols=[b'h2'] - ) + self.url = urlparse(self.mockserver.http_address) self.protocol = H2ClientProtocol() - connectProtocol( - endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), - protocol=self.protocol - ) + # Connect to the server using the custom HTTP2ClientProtocol + options = CertificateOptions(acceptableProtocols=[b'h2']) + endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) + connectProtocol(endpoint, self.protocol) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) - @defer.inlineCallbacks def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.path_temp) + self.mockserver.__exit__(None, None, None) def test_download(self): - request = Request(self.getURL('file')) + request = Request(self.getURL('')) + + def assert_response(response: Response): + self.assertEqual(response.body, b'Scrapy mock HTTP server\n') + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + d = self.protocol.request(request) - d.addCallback(lambda response: response.body) - d.addCallback(self.assertEqual, b"0123456789") + d.addCallback(assert_response) return d