diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6647ae0b7..0b872f6ba 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,6 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI +from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened streams. This counter - # is used to make sure that not more than MAX_CONCURRENT_STREAMS - # streams are opened which leads to ProtocolError - # We use simple FIFO policy to handle pending requests - self._active_streams = 0 - - # Flag to keep track if settings were acknowledged by the remote - # This ensures that we have established a HTTP/2 connection - self._settings_acknowledged = False - # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] - self.metadata: H2ConnectionMetadataDict = { + # Some meta data of this connection + # initialized when connection is successfully made + self.metadata: Dict = { + # Peer certificate instance 'certificate': None, + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully 'ip_address': None, + + # URI of the peer HTTP/2 connection is made 'uri': uri, + + # Both ip_address and uri are used by the Stream before + # initiating the request to verify that the base address + + # Variables taken from Project Settings 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), + + # Counter to keep track of opened streams. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + 'active_streams': 0, + + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + 'settings_acknowledged': False, } @property @@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) and self._settings_acknowledged + return bool(self.transport.connected) and self.metadata['settings_acknowledged'] @property def allowed_max_concurrent_streams(self) -> int: @@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ while ( self._pending_request_stream_pool - and self._active_streams < self.allowed_max_concurrent_streams + and self.metadata['active_streams'] < self.allowed_max_concurrent_streams and self.h2_connected ): - self._active_streams += 1 + self.metadata['active_streams'] += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() self._write_to_transport() @@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Perform cleanup when a stream is closed """ stream = self.streams.pop(stream_id) - self._active_streams -= 1 + self.metadata['active_streams'] -= 1 self._send_pending_requests() return stream @@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if ( self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0 - or self._active_streams > 0 + or self.metadata['active_streams'] > 0 ): error_code = ErrorCodes.PROTOCOL_ERROR else: @@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): close_reason = StreamCloseReason.INACTIVE stream.close(close_reason, self._conn_lost_errors, from_protocol=True) - self._active_streams -= len(self.streams) + self.metadata['active_streams'] -= len(self.streams) self.streams.clear() self._pending_request_stream_pool.clear() self.conn.close_connection() @@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: - self._settings_acknowledged = True + self.metadata['settings_acknowledged'] = True # Send off all the pending requests as now we have # established a proper HTTP/2 connection diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index df7470e11..e01e76ae5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,6 @@ import logging from enum import Enum from io import BytesIO -from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes @@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +from typing import Dict +from typing import List, Optional, Tuple, TYPE_CHECKING -from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -103,21 +103,40 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.metadata: H2StreamMetadataDict = { + # Metadata of an HTTP/2 connection stream + # initialized when stream is instantiated + self.metadata: Dict = { 'request_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether the stream has initiated the request 'request_sent': False, + + # Flag to track whether we have logged about exceeding download warnsize 'reached_warnsize': False, + + # Each time we send a data frame, we will decrease value by the amount send. 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether client (self) have closed this stream 'stream_closed_local': False, + + # Flag to keep track whether the server has closed the stream 'stream_closed_server': False, } # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: H2ResponseDict = { + self._response: Dict = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. 'body': BytesIO(), + + # The amount of data received that counts against the + # flow control window 'flow_controlled_size': 0, + + # Headers received after sending the request 'headers': Headers({}), } diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py deleted file mode 100644 index ff8d94066..000000000 --- a/scrapy/core/http2/types.py +++ /dev/null @@ -1,67 +0,0 @@ -from io import BytesIO -from ipaddress import IPv4Address, IPv6Address -from typing import Union, Optional - -from twisted.internet.ssl import Certificate -from twisted.web.client import URI -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict - -from scrapy.http.headers import Headers - - -class H2ConnectionMetadataDict(TypedDict): - """Some meta data of this connection - initialized when connection is successfully made - """ - certificate: Optional[Certificate] - - # Address of the server we are connected to which - # is updated when HTTP/2 connection is made successfully - ip_address: Optional[Union[IPv4Address, IPv6Address]] - - # URI of the peer HTTP/2 connection is made - uri: URI - - # Both ip_address and uri are used by the Stream before - # initiating the request to verify that the base address - - # Variables taken from Project Settings - default_download_maxsize: int - default_download_warnsize: int - - -class H2StreamMetadataDict(TypedDict): - """Metadata of an HTTP/2 connection stream - initialized when stream is instantiated - """ - - request_content_length: int - - # Flag to keep track whether the stream has initiated the request - request_sent: bool - - # Flag to track whether we have logged about exceeding download warnsize - reached_warnsize: bool - - # Each time we send a data frame, we will decrease value by the amount send. - remaining_content_length: int - - # Flag to keep track whether we have closed this stream - stream_closed_local: bool - - # Flag to keep track whether the server has closed the stream - stream_closed_server: bool - - -class H2ResponseDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers diff --git a/setup.py b/setup.py index 39482d383..34af7ec67 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 0a88ed8af..78090d6de 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,6 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt