From 69f6d038c0bc51a9de706890621d2ce183f79e09 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 07:06:32 +0530 Subject: [PATCH] feat: TypedDict for Stream._response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove test_protocol.py as working testing environment is setup 🙂🙃 - Add typing_extensions as dependency to support TypedDict for python<3.8 --- scrapy/core/http2/protocol.py | 12 +----- scrapy/core/http2/stream.py | 11 ++++- scrapy/core/http2/test_protocol.py | 67 ------------------------------ setup.py | 4 +- 4 files changed, 15 insertions(+), 79 deletions(-) delete mode 100644 scrapy/core/http2/test_protocol.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 455d8777e..9b8ec6c77 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) - -from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request @@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__) class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing - # 2. Add support for cookies - # 3. Handle priority updates (Not required) - # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) - # 1 & 2: - # - Automatically handled by the Request middleware - # - request.headers will have 'Set-Cookie' value + # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): - # TODO: event.stream_id was abruptly closed - # Q. What should be the response? (Failure/Partial/???) self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 112ce5bcd..023f4f4eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +class _ResponseTypedDict(TypedDict): + body: BytesIO + flow_controlled_size: int + headers: Headers + + LOGGER = logging.getLogger(__name__) @@ -100,7 +109,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response = { + self._response: _ResponseTypedDict = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. 'body': BytesIO(), diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py deleted file mode 100644 index c7782a518..000000000 --- a/scrapy/core/http2/test_protocol.py +++ /dev/null @@ -1,67 +0,0 @@ -# This is simple script to test - -import json - -from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS - -from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response, JsonRequest - -try: - with open('data.json', 'r') as f: - JSON_DATA = json.load(f) -except: - JSON_DATA = { - "data": "To test for really large amount of data -- Add data.json with lots of data.", - "why": "To test whether correct data is sent :)" - } - -# Use nghttp2 for testing whether basic setup works - for small response -HTTPBIN_AUTHORITY = u'nghttp2.org' -HTTPBIN_REQUEST_URLS = 1 * [ - Request(url='https://nghttp2.org/httpbin/get', method='GET'), - Request(url='https://nghttp2.org/httpbin/post', method='POST'), - JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), -] - -# Use POKE_API for testing large responses -POKE_API_AUTHORITY = u'pokeapi.co' -POKE_API_REQUESTS = 15 * [ - Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 -] - -AUTHORITY = POKE_API_AUTHORITY -REQUEST_URLS = POKE_API_REQUESTS - -options = optionsForClientTLS( - hostname=AUTHORITY, - acceptableProtocols=[b'h2'], -) - -protocol = H2ClientProtocol() - -count_responses = 1 - - -def print_response(response): - global count_responses - assert isinstance(response, Response) - print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) - count_responses = count_responses + 1 - - -for request in REQUEST_URLS: - d = protocol.request(request) - d.addCallback(print_response) - -connectProtocol( - SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), - protocol -) - -reactor.run() diff --git a/setup.py b/setup.py index dafa5684a..d1470df5e 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5', + python_requires='>=3.5.2', install_requires=[ 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' @@ -80,6 +80,8 @@ setup( 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', + 'itemadapter>=0.1.0', + 'typing_extensions>=3.7' ], extras_require=extras_require, )