diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4a465168f..b95cb05ae 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,8 +6,9 @@ from h2.events import ( ) from scrapy.http import Request +from scrapy.core.http2.stream import Stream -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol from urllib.parse import urlparse @@ -40,31 +41,55 @@ class IH2EventsHandler(Interface): @implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): + # TODO: Check for user-agent while testing + # TODO: Add support for cookies + # TODO: Handle priority updates + def __init__(self): config = H2Configuration(client_side=True) self.conn = H2Connection(config=config) - - # List of ongoing stream id's - self.streams = [] + + # ID of the next request stream + # Assuming each request stream creates a new response stream + # we increment by 2 for each new request stream created + self.next_stream_id = 1 + + # Streams are stored in a dictionary keyed off their stream IDs + self.streams = {} + + def _new_stream(self, headers): + """Instantiates a new Stream object + """ + stream = Stream(self.next_stream_id, headers) + + self.next_stream_id += 2 + + return stream def request(self, _request: Request): + """ + + Arguments: + _request {Request} -- [description] + """ url = urlparse(_request.url) - request_headers = [ - (':method', _request.method), - (':authority', url.netloc), - (':scheme', url.scheme), - (':path', url.path), - ] - - # TODO: Check for user-agent while testing - request_headers += list(_request.headers.items()) - - # TODO: Add support for cookies here + _request[":method"] = _request.method + # TODO: Make authority private class variable instead + # of parsing it from request url all requests to same + # host are multiplexed into one connection & a connection + # can have only 1 host at a time + _request[":authority"] = url.netloc + # TODO: Check if scheme can be 'http' for HTTP/2 ? + _request[":scheme"] = "https" + _request[":path"] = url.path + stream = self._new_stream(_request.headers) + d = stream.get_response() + return d def connectionMade(self): @@ -76,7 +101,6 @@ class H2ClientProtocol(Protocol): def dataReceived(self, data): events = self.conn.receive_data(data) - self._handle_events(events) _data = self.conn.data_to_send() @@ -85,9 +109,10 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. - """ + """ - for stream_id in self.streams: + for stream_id in self.streams.keys(): + # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) def _handle_events(self, events): @@ -114,23 +139,23 @@ class H2ClientProtocol(Protocol): elif isinstance(event, WindowUpdated): self.window_updated(event) - def connection_terminated(self, event): + def connection_terminated(self, event: ConnectionTerminated): pass - def data_received(self, event): + def data_received(self, event: DataReceived): pass - def response_received(self, event): + def response_received(self, event: ResponseReceived): pass - def stream_ended(self, event): + def stream_ended(self, event: StreamEnded): pass - def stream_reset(self, event): + def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event): + def trailers_received(self, event: TrailersReceived): pass - def window_updated(self, event): + def window_updated(self, event: WindowUpdated): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py new file mode 100644 index 000000000..558a6552c --- /dev/null +++ b/scrapy/core/http2/stream.py @@ -0,0 +1,44 @@ +from scrapy.http.headers import Headers + + +class Stream: + """Represents a single HTTP/2 Stream. + + Stream is a bidirectional flow of bytes within an established connection, + which may carry one or more messages. Handles the tranfer of HTTP Headers + and Data frames. + """ + + def __init__(self, stream_id, headers): + """ + Arguments: + stream_id {int} -- For one HTTP/2 connection each stream is + uniquely identified by a single integer + headers {Headers} -- HTTP request headers + """ + + # Headers received after sending the request + self.response_headers = Headers({}) + + # Headers which are send with the request + # These cannot be modified any furthur + self._request_headers = headers + + # TODO: Add canceller for the Deferred below + self._deferred_response = Deferred() + + def get_response(self): + """Simply return a Deferred which fires when response + from the asynchronous request is available + + Returns: + Deferred -- Calls the callback when the response is + avaialble + """ + return self._deferred_response + + def receive_data(self, data): + pass + + def receive_headers(self, headers): + pass