RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: remove typing-extensions dependency

This commit is contained in:
Aditya 2020-08-27 15:12:22 +05:30
parent 450ba6b51f
commit 5e36f539e2
5 changed files with 55 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
scrapy/core/http2/protocol.py
View File

@ -3,7 +3,6 @@ import itertools
import logging
from collections import deque
from ipaddress import IPv4Address, IPv6Address
from typing import Dict, List, Optional, Union
from h2.config import H2Configuration
from h2.connection import H2Connection
@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from twisted.python.failure import Failure
from twisted.web.client import URI
from typing import Dict, List, Optional, Union
from zope.interface import implementer
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.core.http2.types import H2ConnectionMetadataDict
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# all requests in a pool and send them as the connection is made
self._pending_request_stream_pool: deque = deque()
# Counter to keep track of opened streams. This counter
# is used to make sure that not more than MAX_CONCURRENT_STREAMS
# streams are opened which leads to ProtocolError
# We use simple FIFO policy to handle pending requests
self._active_streams = 0
# Flag to keep track if settings were acknowledged by the remote
# This ensures that we have established a HTTP/2 connection
self._settings_acknowledged = False
# Save an instance of errors raised which lead to losing the connection
# We pass these instances to the streams ResponseFailed() failure
self._conn_lost_errors: List[BaseException] = []
self.metadata: H2ConnectionMetadataDict = {
# Some meta data of this connection
# initialized when connection is successfully made
self.metadata: Dict = {
# Peer certificate instance
'certificate': None,
# Address of the server we are connected to which
# is updated when HTTP/2 connection is made successfully
'ip_address': None,
# URI of the peer HTTP/2 connection is made
'uri': uri,
# Both ip_address and uri are used by the Stream before
# initiating the request to verify that the base address
# Variables taken from Project Settings
'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
# Counter to keep track of opened streams. This counter
# is used to make sure that not more than MAX_CONCURRENT_STREAMS
# streams are opened which leads to ProtocolError
# We use simple FIFO policy to handle pending requests
'active_streams': 0,
# Flag to keep track if settings were acknowledged by the remote
# This ensures that we have established a HTTP/2 connection
'settings_acknowledged': False,
}
@property
@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
This is used while initiating pending streams to make sure
that we initiate stream only during active HTTP/2 Connection
"""
return bool(self.transport.connected) and self._settings_acknowledged
return bool(self.transport.connected) and self.metadata['settings_acknowledged']
@property
def allowed_max_concurrent_streams(self) -> int:
@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""
while (
self._pending_request_stream_pool
and self._active_streams < self.allowed_max_concurrent_streams
and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
and self.h2_connected
):
self._active_streams += 1
self.metadata['active_streams'] += 1
stream = self._pending_request_stream_pool.popleft()
stream.initiate_request()
self._write_to_transport()
@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""Perform cleanup when a stream is closed
"""
stream = self.streams.pop(stream_id)
self._active_streams -= 1
self.metadata['active_streams'] -= 1
self._send_pending_requests()
return stream
@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
if (
self.conn.open_outbound_streams > 0
or self.conn.open_inbound_streams > 0
or self._active_streams > 0
or self.metadata['active_streams'] > 0
):
error_code = ErrorCodes.PROTOCOL_ERROR
else:
@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
close_reason = StreamCloseReason.INACTIVE
stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
self._active_streams -= len(self.streams)
self.metadata['active_streams'] -= len(self.streams)
self.streams.clear()
self._pending_request_stream_pool.clear()
self.conn.close_connection()
@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self.streams[event.stream_id].receive_headers(event.headers)
def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
self._settings_acknowledged = True
self.metadata['settings_acknowledged'] = True
# Send off all the pending requests as now we have
# established a proper HTTP/2 connection

27
scrapy/core/http2/stream.py
View File

@ -1,7 +1,6 @@
import logging
from enum import Enum
from io import BytesIO
from typing import List, Optional, Tuple, TYPE_CHECKING
from urllib.parse import urlparse
from h2.errors import ErrorCodes
@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
from typing import Dict
from typing import List, Optional, Tuple, TYPE_CHECKING
from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
@ -103,21 +103,40 @@ class Stream:
self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
self.metadata: H2StreamMetadataDict = {
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
self.metadata: Dict = {
'request_content_length': 0 if self._request.body is None else len(self._request.body),
# Flag to keep track whether the stream has initiated the request
'request_sent': False,
# Flag to track whether we have logged about exceeding download warnsize
'reached_warnsize': False,
# Each time we send a data frame, we will decrease value by the amount send.
'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
# Flag to keep track whether client (self) have closed this stream
'stream_closed_local': False,
# Flag to keep track whether the server has closed the stream
'stream_closed_server': False,
}
# Private variable used to build the response
# this response is then converted to appropriate Response class
# passed to the response deferred callback
self._response: H2ResponseDict = {
self._response: Dict = {
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
'body': BytesIO(),
# The amount of data received that counts against the
# flow control window
'flow_controlled_size': 0,
# Headers received after sending the request
'headers': Headers({}),
}

67
scrapy/core/http2/types.py
View File

@ -1,67 +0,0 @@
from io import BytesIO
from ipaddress import IPv4Address, IPv6Address
from typing import Union, Optional
from twisted.internet.ssl import Certificate
from twisted.web.client import URI
# for python < 3.8 -- typing.TypedDict is undefined
from typing_extensions import TypedDict
from scrapy.http.headers import Headers
class H2ConnectionMetadataDict(TypedDict):
"""Some meta data of this connection
initialized when connection is successfully made
"""
certificate: Optional[Certificate]
# Address of the server we are connected to which
# is updated when HTTP/2 connection is made successfully
ip_address: Optional[Union[IPv4Address, IPv6Address]]
# URI of the peer HTTP/2 connection is made
uri: URI
# Both ip_address and uri are used by the Stream before
# initiating the request to verify that the base address
# Variables taken from Project Settings
default_download_maxsize: int
default_download_warnsize: int
class H2StreamMetadataDict(TypedDict):
"""Metadata of an HTTP/2 connection stream
initialized when stream is instantiated
"""
request_content_length: int
# Flag to keep track whether the stream has initiated the request
request_sent: bool
# Flag to track whether we have logged about exceeding download warnsize
reached_warnsize: bool
# Each time we send a data frame, we will decrease value by the amount send.
remaining_content_length: int
# Flag to keep track whether we have closed this stream
stream_closed_local: bool
# Flag to keep track whether the server has closed the stream
stream_closed_server: bool
class H2ResponseDict(TypedDict):
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
body: BytesIO
# The amount of data received that counts against the flow control
# window
flow_controlled_size: int
# Headers received after sending the request
headers: Headers

1
setup.py
View File

@ -33,7 +33,6 @@ install_requires = [
'protego>=0.1.15',
'itemadapter>=0.1.0',
'h2>=3.2.0',
'typing-extensions>=3.7.4',
]
extras_require = {}

1
tox.ini
View File

@ -74,7 +74,6 @@ deps =
queuelib==1.4.2
service_identity==16.0.0
Twisted[http2]==17.9.0
typing-extensions==3.7.4
w3lib==1.17.0
zope.interface==4.1.3
-rtests/requirements-py3.txt