test: mockserver with h2 protocol for tests

- add Twisted[http2] in setup.py requirements
- add test_protocol.py to test the current implementation

BREAKING CHANGES
test_download times out because of no protocol negotiated between
Mockserver and HTTP/2 client

scrapy/core/http2/test_protocol.py

@@ -0,0 +1,67 @@
+# This is simple script to test
+
+import json
+
+from twisted.internet import reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request, Response, JsonRequest
+
+try:
+    with open('data.json', 'r') as f:
+        JSON_DATA = json.load(f)
+except:
+    JSON_DATA = {
+        "data": "To test for really large amount of data -- Add data.json with lots of data.",
+        "why": "To test whether correct data is sent :)"
+    }
+
+# Use nghttp2 for testing whether basic setup works - for small response
+HTTPBIN_AUTHORITY = u'nghttp2.org'
+HTTPBIN_REQUEST_URLS = 1 * [
+    Request(url='https://nghttp2.org/httpbin/get', method='GET'),
+    Request(url='https://nghttp2.org/httpbin/post', method='POST'),
+    JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
+]
+
+# Use POKE_API for testing large responses
+POKE_API_AUTHORITY = u'pokeapi.co'
+POKE_API_REQUESTS = 15 * [
+    Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'),  # should give 404
+]
+
+AUTHORITY = POKE_API_AUTHORITY
+REQUEST_URLS = POKE_API_REQUESTS
+
+options = optionsForClientTLS(
+    hostname=AUTHORITY,
+    acceptableProtocols=[b'h2'],
+)
+
+protocol = H2ClientProtocol()
+
+count_responses = 1
+
+
+def print_response(response):
+    global count_responses
+    assert isinstance(response, Response)
+    print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
+    count_responses = count_responses + 1
+
+
+for request in REQUEST_URLS:
+    d = protocol.request(request)
+    d.addCallback(print_response)
+
+connectProtocol(
+    SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
+    protocol
+)
+
+reactor.run()

setup.py

@@ -1,8 +1,8 @@
 from os.path import dirname, join
+
 from pkg_resources import parse_version
 from setuptools import setup, find_packages, __version__ as setuptools_version
 
-
 with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
@@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 
-
 setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
-    project_urls = {
+    project_urls={
         'Documentation': 'https://docs.scrapy.org/',
         'Source': 'https://github.com/scrapy/scrapy',
         'Tracker': 'https://github.com/scrapy/scrapy/issues',
@@ -69,6 +68,7 @@ setup(
     python_requires='>=3.5',
     install_requires=[
         'Twisted>=17.9.0',
+        'Twisted[http2]>=17.9.0'
         'cryptography>=2.0',
         'cssselect>=0.9.1',
         'lxml>=3.5.0',

tests/test_http2_client_protocol.py

@@ -0,0 +1,71 @@
+import os
+import shutil
+
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+from twisted.protocols.policies import WrappingFactory
+from twisted.python.filepath import FilePath
+from twisted.trial import unittest
+from twisted.web import static, server
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request
+from tests.mockserver import ssl_context_factory
+
+
+class Http2ClientProtocolTestCase(unittest.TestCase):
+    scheme = 'https'
+
+    # only used for HTTPS tests
+    file_key = 'keys/localhost.key'
+    file_certificate = 'keys/localhost.crt'
+
+    def setUp(self):
+        # Start server for testing
+        self.path_temp = self.mktemp()
+        os.mkdir(self.path_temp)
+        FilePath(self.path_temp).child('file').setContent(b"0123456789")
+        r = static.File(self.path_temp)
+
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.host = 'localhost'
+        if self.scheme is 'https':
+            self.port = reactor.listenSSL(
+                0, self.wrapper,
+                ssl_context_factory(self.file_key, self.file_certificate),
+                interface=self.host
+            )
+        else:
+            self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
+
+        self.port_number = self.port.getHost().port
+
+        # Connect to the server using the custom HTTP2ClientProtocol
+        options = optionsForClientTLS(
+            hostname=self.host,
+            acceptableProtocols=[b'h2']
+        )
+
+        self.protocol = H2ClientProtocol()
+
+        connectProtocol(
+            endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
+            protocol=self.protocol
+        )
+
+    def getURL(self, path):
+        return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        shutil.rmtree(self.path_temp)
+
+    def test_download(self):
+        request = Request(self.getURL('file'))
+        d = self.protocol.request(request)
+        d.addCallback(lambda response: response.body)
+        d.addCallback(self.assertEqual, b"0123456789")
+        return d