From 700df3eeb7c540892c8b2910db688f1cc5d1c845 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 17 Jun 2020 21:02:14 +0530 Subject: [PATCH] test: mockserver with h2 protocol for tests - add Twisted[http2] in setup.py requirements - add test_protocol.py to test the current implementation BREAKING CHANGES test_download times out because of no protocol negotiated between Mockserver and HTTP/2 client --- scrapy/core/http2/test_protocol.py | 67 +++++++++++++++++++++++++++ setup.py | 6 +-- tests/test_http2_client_protocol.py | 71 +++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 scrapy/core/http2/test_protocol.py create mode 100644 tests/test_http2_client_protocol.py diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py new file mode 100644 index 000000000..c7782a518 --- /dev/null +++ b/scrapy/core/http2/test_protocol.py @@ -0,0 +1,67 @@ +# This is simple script to test + +import json + +from twisted.internet import reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request, Response, JsonRequest + +try: + with open('data.json', 'r') as f: + JSON_DATA = json.load(f) +except: + JSON_DATA = { + "data": "To test for really large amount of data -- Add data.json with lots of data.", + "why": "To test whether correct data is sent :)" + } + +# Use nghttp2 for testing whether basic setup works - for small response +HTTPBIN_AUTHORITY = u'nghttp2.org' +HTTPBIN_REQUEST_URLS = 1 * [ + Request(url='https://nghttp2.org/httpbin/get', method='GET'), + Request(url='https://nghttp2.org/httpbin/post', method='POST'), + JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), +] + +# Use POKE_API for testing large responses +POKE_API_AUTHORITY = u'pokeapi.co' +POKE_API_REQUESTS = 15 * [ + Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 +] + +AUTHORITY = POKE_API_AUTHORITY +REQUEST_URLS = POKE_API_REQUESTS + +options = optionsForClientTLS( + hostname=AUTHORITY, + acceptableProtocols=[b'h2'], +) + +protocol = H2ClientProtocol() + +count_responses = 1 + + +def print_response(response): + global count_responses + assert isinstance(response, Response) + print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) + count_responses = count_responses + 1 + + +for request in REQUEST_URLS: + d = protocol.request(request) + d.addCallback(print_response) + +connectProtocol( + SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), + protocol +) + +reactor.run() diff --git a/setup.py b/setup.py index 1b3c6771a..dafa5684a 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join + from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version - with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] - setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -69,6 +68,7 @@ setup( python_requires='>=3.5', install_requires=[ 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py new file mode 100644 index 000000000..7830f7028 --- /dev/null +++ b/tests/test_http2_client_protocol.py @@ -0,0 +1,71 @@ +import os +import shutil + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.policies import WrappingFactory +from twisted.python.filepath import FilePath +from twisted.trial import unittest +from twisted.web import static, server + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request +from tests.mockserver import ssl_context_factory + + +class Http2ClientProtocolTestCase(unittest.TestCase): + scheme = 'https' + + # only used for HTTPS tests + file_key = 'keys/localhost.key' + file_certificate = 'keys/localhost.crt' + + def setUp(self): + # Start server for testing + self.path_temp = self.mktemp() + os.mkdir(self.path_temp) + FilePath(self.path_temp).child('file').setContent(b"0123456789") + r = static.File(self.path_temp) + + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = 'localhost' + if self.scheme is 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, + ssl_context_factory(self.file_key, self.file_certificate), + interface=self.host + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + + self.port_number = self.port.getHost().port + + # Connect to the server using the custom HTTP2ClientProtocol + options = optionsForClientTLS( + hostname=self.host, + acceptableProtocols=[b'h2'] + ) + + self.protocol = H2ClientProtocol() + + connectProtocol( + endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), + protocol=self.protocol + ) + + def getURL(self, path): + return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.path_temp) + + def test_download(self): + request = Request(self.getURL('file')) + d = self.protocol.request(request) + d.addCallback(lambda response: response.body) + d.addCallback(self.assertEqual, b"0123456789") + return d