test: GET request for HTTP2Client using mockserver

scrapy/core/http2/protocol.py

@@ -9,6 +9,8 @@ from h2.events import (
     DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
+
+from twisted.internet.ssl import Certificate
 from twisted.internet.protocol import connectionDone, Protocol
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
@@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol):
         self.destination = self.transport.getPeer()
         LOGGER.info('Connection made to {}'.format(self.destination))
 
-        self._metadata['certificate'] = self.transport.getPeerCertificate()
+        self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
         self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
 
         self.conn.initiate_connection()

scrapy/core/http2/stream.py

@@ -117,7 +117,7 @@ class Stream:
         self._deferred_response = Deferred()
 
     def __str__(self):
-        return "Stream(id={})".format(self.stream_id)
+        return "Stream(id={})".format(repr(self.stream_id))
 
     __repr__ = __str__
 
@@ -299,9 +299,6 @@ class Stream:
     def close(self, reason: StreamCloseReason):
         """Based on the reason sent we will handle each case.
         """
-        # TODO: In case of abruptly stream close
-        #  Q1. Do we need to send the request again?
-        #  Q2. What response should we send now?
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
 
@@ -346,10 +343,7 @@ class Stream:
         and fires the response deferred callback with the
         generated response instance"""
         # TODO:
-        #  2. Should we fire this in case of
-        #   2.1 StreamReset in between when data is received partially
-        #   2.2 Forcefully closed the stream
-        #  3. Update Client Side Status Codes here
+        #  1. Update Client Side Status Codes here
 
         response_cls = responsetypes.from_args(
             headers=self._response['headers'],

tests/test_http2_client_protocol.py

@@ -1,71 +1,50 @@
-import os
-import shutil
+from urllib.parse import urlparse
 
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
 from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-from twisted.protocols.policies import WrappingFactory
-from twisted.python.filepath import FilePath
+from twisted.internet.ssl import CertificateOptions
 from twisted.trial import unittest
-from twisted.web import static, server
 
 from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request
-from tests.mockserver import ssl_context_factory
+from scrapy.http import Request, Response
+from tests.mockserver import MockServer
 
 
 class Http2ClientProtocolTestCase(unittest.TestCase):
     scheme = 'https'
 
-    # only used for HTTPS tests
-    file_key = 'keys/localhost.key'
-    file_certificate = 'keys/localhost.crt'
-
     def setUp(self):
         # Start server for testing
-        self.path_temp = self.mktemp()
-        os.mkdir(self.path_temp)
-        FilePath(self.path_temp).child('file').setContent(b"0123456789")
-        r = static.File(self.path_temp)
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
 
-        self.site = server.Site(r, timeout=None)
-        self.wrapper = WrappingFactory(self.site)
-        self.host = 'localhost'
-        if self.scheme is 'https':
-            self.port = reactor.listenSSL(
-                0, self.wrapper,
-                ssl_context_factory(self.file_key, self.file_certificate),
-                interface=self.host
-            )
+        if self.scheme == 'https':
+            self.url = urlparse(self.mockserver.https_address)
         else:
-            self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
-
-        self.port_number = self.port.getHost().port
-
-        # Connect to the server using the custom HTTP2ClientProtocol
-        options = optionsForClientTLS(
-            hostname=self.host,
-            acceptableProtocols=[b'h2']
-        )
+            self.url = urlparse(self.mockserver.http_address)
 
         self.protocol = H2ClientProtocol()
 
-        connectProtocol(
-            endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
-            protocol=self.protocol
-        )
+        # Connect to the server using the custom HTTP2ClientProtocol
+        options = CertificateOptions(acceptableProtocols=[b'h2'])
+        endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options)
+        connectProtocol(endpoint, self.protocol)
 
     def getURL(self, path):
-        return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+        return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path)
 
-    @defer.inlineCallbacks
     def tearDown(self):
-        yield self.port.stopListening()
-        shutil.rmtree(self.path_temp)
+        self.mockserver.__exit__(None, None, None)
 
     def test_download(self):
-        request = Request(self.getURL('file'))
+        request = Request(self.getURL(''))
+
+        def assert_response(response: Response):
+            self.assertEqual(response.body, b'Scrapy mock HTTP server\n')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+
         d = self.protocol.request(request)
-        d.addCallback(lambda response: response.body)
-        d.addCallback(self.assertEqual, b"0123456789")
+        d.addCallback(assert_response)
         return d