RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HTTP/2: add some type hints (#4785)

This commit is contained in:
Eugenio Lacuesta 2020-09-16 04:57:07 -03:00 committed by GitHub
parent 4d6359df2d
commit 6e8d20a07a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrapy/core/downloader/handlers/http2.py
View File

@ -1,8 +1,9 @@
import warnings
from time import time
from typing import Optional
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
@ -10,14 +11,18 @@ from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler=None):
def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
self._crawler = crawler
from twisted.internet import reactor
@ -25,14 +30,14 @@ class H2DownloadHandler:
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
crawler=self._crawler
crawler=self._crawler,
)
return agent.download_request(request, spider)
@ -47,8 +52,9 @@ class ScrapyH2Agent:
def __init__(
self, context_factory,
pool: H2ConnectionPool,
connect_timeout=10, bind_address: Optional[bytes] = None,
crawler=None
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,
crawler: Optional[Crawler] = None,
) -> None:
self._context_factory = context_factory
self._connect_timeout = connect_timeout
@ -80,7 +86,7 @@ class ScrapyH2Agent:
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
pool=self._pool,
)
return self._Agent(
@ -88,7 +94,7 @@ class ScrapyH2Agent:
context_factory=self._context_factory,
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
pool=self._pool,
)
def download_request(self, request: Request, spider: Spider) -> Deferred:
@ -110,7 +116,7 @@ class ScrapyH2Agent:
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response:
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
if timeout_cl.active():
timeout_cl.cancel()
return response

27
scrapy/core/http2/agent.py
View File

@ -1,5 +1,5 @@
from collections import deque
from typing import Deque, Dict, List, Tuple, Optional
from typing import Deque, Dict, List, Optional, Tuple
from twisted.internet import defer
from twisted.internet.base import ReactorBase
@ -95,16 +95,18 @@ class H2ConnectionPool:
class H2Agent:
def __init__(
self, reactor: ReactorBase, pool: H2ConnectionPool,
context_factory=BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
self,
reactor: ReactorBase,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
self._reactor = reactor
self._pool = pool
self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
self.endpoint_factory = _StandardEndpointFactory(
self._reactor, self._context_factory,
connect_timeout, bind_address
self._reactor, self._context_factory, connect_timeout, bind_address
)
def get_endpoint(self, uri: URI):
@ -132,17 +134,20 @@ class H2Agent:
class ScrapyProxyH2Agent(H2Agent):
def __init__(
self, reactor: ReactorBase,
proxy_uri: URI, pool: H2ConnectionPool,
context_factory=BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
self,
reactor: ReactorBase,
proxy_uri: URI,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,
connect_timeout=connect_timeout,
bind_address=bind_address
bind_address=bind_address,
)
self._proxy_uri = proxy_uri

15
scrapy/core/http2/protocol.py
View File

@ -29,6 +29,7 @@ from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error):
class RemoteTerminatedConnection(H2Error):
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated):
def __init__(
self,
remote_ip_address: Optional[Union[IPv4Address, IPv6Address]],
event: ConnectionTerminated,
) -> None:
self.remote_ip_address = remote_ip_address
self.terminate_event = event
@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error):
class MethodNotAllowed405(H2Error):
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]):
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
self.remote_ip_address = remote_ip_address
def __str__(self) -> str:
@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self.conn.initiate_connection()
self._write_to_transport()
def _lose_connection_with_error(self, errors: List[BaseException]):
def _lose_connection_with_error(self, errors: List[BaseException]) -> None:
"""Helper function to lose the connection with the error sent as a
reason"""
self._conn_lost_errors += errors
self.transport.loseConnection()
def handshakeCompleted(self):
def handshakeCompleted(self) -> None:
"""We close the connection with InvalidNegotiatedProtocol exception
when the connection was not made via h2 protocol"""
negotiated_protocol = self.transport.negotiatedProtocol
@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
finally:
self._write_to_transport()
def timeoutConnection(self):
def timeoutConnection(self) -> None:
"""Called when the connection times out.
We lose the connection with TimeoutError"""

7
scrapy/core/http2/stream.py
View File

@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes
if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
logger = logging.getLogger(__name__)
@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed):
of the stream. This happens when a stream is waiting for other
streams to close and connection is lost."""
def __init__(self, request: Request):
def __init__(self, request: Request) -> None:
self.request = request
def __str__(self) -> str:
@ -139,7 +140,7 @@ class Stream:
'headers': Headers({}),
}
def _cancel(_):
def _cancel(_) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
@ -360,7 +361,7 @@ class Stream:
self,
reason: StreamCloseReason,
errors: Optional[List[BaseException]] = None,
from_protocol: bool = False
from_protocol: bool = False,
) -> None:
"""Based on the reason sent we will handle each case.
"""

6
setup.cfg
View File

@ -82,9 +82,6 @@ ignore_errors = True
[mypy-tests.test_downloader_handlers]
ignore_errors = True
[mypy-tests.test_downloader_handlers_http2]
ignore_errors = True
[mypy-tests.test_engine]
ignore_errors = True
@ -94,9 +91,6 @@ ignore_errors = True
[mypy-tests.test_http_request]
ignore_errors = True
[mypy-tests.test_http2_client_protocol]
ignore_errors = True
[mypy-tests.test_linkextractors]
ignore_errors = True

19
tests/test_downloader_handlers.py
View File

@ -2,6 +2,7 @@ import contextlib
import os
import shutil
import tempfile
from typing import Optional, Type
from unittest import mock
from testfixtures import LogCapture
@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource):
class HttpTestCase(unittest.TestCase):
scheme = 'http'
download_handler_cls = HTTPDownloadHandler
download_handler_cls: Type = HTTPDownloadHandler
# only used for HTTPS tests
keyfile = 'keys/localhost.key'
@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase):
class Http10TestCase(HttpTestCase):
"""HTTP 1.0 test case"""
download_handler_cls = HTTP10DownloadHandler
download_handler_cls: Type = HTTP10DownloadHandler
class Https10TestCase(Http10TestCase):
@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase):
class Http11TestCase(HttpTestCase):
"""HTTP 1.1 test case"""
download_handler_cls = HTTP11DownloadHandler
download_handler_cls: Type = HTTP11DownloadHandler
def test_download_without_maxsize_limit(self):
request = Request(self.getURL('file'))
@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
class Https11CustomCiphers(unittest.TestCase):
scheme = 'https'
download_handler_cls = HTTP11DownloadHandler
download_handler_cls: Type = HTTP11DownloadHandler
keyfile = 'keys/localhost.key'
certfile = 'keys/localhost.crt'
@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase):
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
settings_dict = None
settings_dict: Optional[dict] = None
def setUp(self):
self.mockserver = MockServer()
@ -668,7 +669,7 @@ class UriResource(resource.Resource):
class HttpProxyTestCase(unittest.TestCase):
download_handler_cls = HTTPDownloadHandler
download_handler_cls: Type = HTTPDownloadHandler
expected_http_proxy_request_body = b'http://example.com'
def setUp(self):
@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase):
class Http10ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP10DownloadHandler
download_handler_cls: Type = HTTP10DownloadHandler
def test_download_with_proxy_https_noconnect(self):
raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
class Http11ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP11DownloadHandler
download_handler_cls: Type = HTTP11DownloadHandler
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase):
class S3TestCase(unittest.TestCase):
download_handler_cls = S3DownloadHandler
download_handler_cls: Type = S3DownloadHandler
# test use same example keys than amazon developer guide
# http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf

2
tests/test_http2_client_protocol.py
View File

@ -45,7 +45,7 @@ def make_html_body(val):
class DummySpider(Spider):
name = 'dummy'
start_urls = []
start_urls: list = []
def parse(self, response):
print(response)