diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4be888bda..e97c31e90 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,8 +1,9 @@ import warnings from time import time -from typing import Optional +from typing import Optional, Type, TypeVar from urllib.parse import urldefrag +from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -10,14 +11,18 @@ from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") + + class H2DownloadHandler: - def __init__(self, settings: Settings, crawler=None): + def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): self._crawler = crawler from twisted.internet import reactor @@ -25,14 +30,14 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - crawler=self._crawler + crawler=self._crawler, ) return agent.download_request(request, spider) @@ -47,8 +52,9 @@ class ScrapyH2Agent: def __init__( self, context_factory, pool: H2ConnectionPool, - connect_timeout=10, bind_address: Optional[bytes] = None, - crawler=None + connect_timeout: int = 10, + bind_address: Optional[bytes] = None, + crawler: Optional[Crawler] = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -80,7 +86,7 @@ class ScrapyH2Agent: proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) return self._Agent( @@ -88,7 +94,7 @@ class ScrapyH2Agent: context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -110,7 +116,7 @@ class ScrapyH2Agent: return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d950c6cfb..a142fa210 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,5 +1,5 @@ from collections import deque -from typing import Deque, Dict, List, Tuple, Optional +from typing import Deque, Dict, List, Optional, Tuple from twisted.internet import defer from twisted.internet.base import ReactorBase @@ -95,16 +95,18 @@ class H2ConnectionPool: class H2Agent: def __init__( - self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self.endpoint_factory = _StandardEndpointFactory( - self._reactor, self._context_factory, - connect_timeout, bind_address + self._reactor, self._context_factory, connect_timeout, bind_address ) def get_endpoint(self, uri: URI): @@ -132,17 +134,20 @@ class H2Agent: class ScrapyProxyH2Agent(H2Agent): def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + proxy_uri: URI, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, context_factory=context_factory, connect_timeout=connect_timeout, - bind_address=bind_address + bind_address=bind_address, ) self._proxy_uri = proxy_uri diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index e32e2b6fe..9d499596c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -29,6 +29,7 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + def __init__( + self, + remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + event: ConnectionTerminated, + ) -> None: self.remote_ip_address = remote_ip_address self.terminate_event = event @@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]): + def _lose_connection_with_error(self, errors: List[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors self.transport.loseConnection() - def handshakeCompleted(self): + def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol @@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): finally: self._write_to_transport() - def timeoutConnection(self): + def timeoutConnection(self) -> None: """Called when the connection times out. We lose the connection with TimeoutError""" diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ef90773b6..3ae2e8db8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol + logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed): of the stream. This happens when a stream is waiting for other streams to close and connection is lost.""" - def __init__(self, request: Request): + def __init__(self, request: Request) -> None: self.request = request def __str__(self) -> str: @@ -139,7 +140,7 @@ class Stream: 'headers': Headers({}), } - def _cancel(_): + def _cancel(_) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method @@ -360,7 +361,7 @@ class Stream: self, reason: StreamCloseReason, errors: Optional[List[BaseException]] = None, - from_protocol: bool = False + from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case. """ diff --git a/setup.cfg b/setup.cfg index 94c1e7b89..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,9 +82,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_downloader_handlers_http2] -ignore_errors = True - [mypy-tests.test_engine] ignore_errors = True @@ -94,9 +91,6 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True -[mypy-tests.test_http2_client_protocol] -ignore_errors = True - [mypy-tests.test_linkextractors] ignore_errors = True diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 20e31f7f7..5b4a2d270 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,6 +2,7 @@ import contextlib import os import shutil import tempfile +from typing import Optional, Type from unittest import mock from testfixtures import LogCapture @@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = 'http' - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler class Https10TestCase(Http10TestCase): @@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = 'https' - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler keyfile = 'keys/localhost.key' certfile = 'keys/localhost.crt' @@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict = None + settings_dict: Optional[dict] = None def setUp(self): self.mockserver = MockServer() @@ -668,7 +669,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler expected_http_proxy_request_body = b'http://example.com' def setUp(self): @@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls = S3DownloadHandler + download_handler_cls: Type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4926ada14..d9ab553f0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -45,7 +45,7 @@ def make_html_body(val): class DummySpider(Spider): name = 'dummy' - start_urls = [] + start_urls: list = [] def parse(self, response): print(response)