feat: TypedDict for Stream._response

- remove test_protocol.py as working testing environment is setup 🙂🙃
- Add typing_extensions as dependency to support TypedDict for
python<3.8

scrapy/core/http2/protocol.py

@@ -9,9 +9,8 @@ from h2.events import (
     DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
-
-from twisted.internet.ssl import Certificate
 from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.ssl import Certificate
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.http import Request
@@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__)
 class H2ClientProtocol(Protocol):
     # TODO:
     #  1. Check for user-agent while testing
-    #  2. Add support for cookies
-    #  3. Handle priority updates (Not required)
-    #  4. Handle case when received events have StreamID = 0 (applied to H2Connection)
-    #  1 & 2:
-    #   - Automatically handled by the Request middleware
-    #   - request.headers will have 'Set-Cookie' value
+    #  2. Handle case when received events have StreamID = 0 (applied to H2Connection)
 
     def __init__(self):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
@@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol):
         self.streams[stream_id].close(StreamCloseReason.ENDED)
 
     def stream_reset(self, event: StreamReset):
-        # TODO: event.stream_id was abruptly closed
-        #  Q. What should be the response? (Failure/Partial/???)
         self.streams[event.stream_id].close(StreamCloseReason.RESET)
 
     def window_updated(self, event: WindowUpdated):

scrapy/core/http2/stream.py

@@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
+# for python < 3.8 -- typing.TypedDict is undefined
+from typing_extensions import TypedDict
 
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
+
+class _ResponseTypedDict(TypedDict):
+    body: BytesIO
+    flow_controlled_size: int
+    headers: Headers
+
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -100,7 +109,7 @@ class Stream:
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
-        self._response = {
+        self._response: _ResponseTypedDict = {
             # Data received frame by frame from the server is appended
             # and passed to the response Deferred when completely received.
             'body': BytesIO(),

scrapy/core/http2/test_protocol.py

@@ -1,67 +0,0 @@
-# This is simple script to test
-
-import json
-
-from twisted.internet import reactor
-from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-
-from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request, Response, JsonRequest
-
-try:
-    with open('data.json', 'r') as f:
-        JSON_DATA = json.load(f)
-except:
-    JSON_DATA = {
-        "data": "To test for really large amount of data -- Add data.json with lots of data.",
-        "why": "To test whether correct data is sent :)"
-    }
-
-# Use nghttp2 for testing whether basic setup works - for small response
-HTTPBIN_AUTHORITY = u'nghttp2.org'
-HTTPBIN_REQUEST_URLS = 1 * [
-    Request(url='https://nghttp2.org/httpbin/get', method='GET'),
-    Request(url='https://nghttp2.org/httpbin/post', method='POST'),
-    JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
-]
-
-# Use POKE_API for testing large responses
-POKE_API_AUTHORITY = u'pokeapi.co'
-POKE_API_REQUESTS = 15 * [
-    Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'),  # should give 404
-]
-
-AUTHORITY = POKE_API_AUTHORITY
-REQUEST_URLS = POKE_API_REQUESTS
-
-options = optionsForClientTLS(
-    hostname=AUTHORITY,
-    acceptableProtocols=[b'h2'],
-)
-
-protocol = H2ClientProtocol()
-
-count_responses = 1
-
-
-def print_response(response):
-    global count_responses
-    assert isinstance(response, Response)
-    print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
-    count_responses = count_responses + 1
-
-
-for request in REQUEST_URLS:
-    d = protocol.request(request)
-    d.addCallback(print_response)
-
-connectProtocol(
-    SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
-    protocol
-)
-
-reactor.run()

setup.py

@@ -65,7 +65,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.5',
+    python_requires='>=3.5.2',
     install_requires=[
         'Twisted>=17.9.0',
         'Twisted[http2]>=17.9.0'
@@ -80,6 +80,8 @@ setup(
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',
+        'itemadapter>=0.1.0',
+        'typing_extensions>=3.7'
     ],
     extras_require=extras_require,
 )