From 045387e07faba4a18ecff919457d69938e89b710 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 19:56:23 +0500 Subject: [PATCH] More typing for scrapy/core/downloader/handlers/http11.py. --- scrapy/core/downloader/handlers/http11.py | 286 +++++++++++++--------- scrapy/core/downloader/handlers/s3.py | 5 +- 2 files changed, 169 insertions(+), 122 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2e7ea559a..40ae1921a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,31 +8,33 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urldefrag, urlunparse -from twisted.internet import defer, protocol, ssl +from twisted.internet import ssl +from twisted.internet.base import ReactorBase +from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from twisted.internet.interfaces import IConsumer +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import ( - URI, - Agent, - HTTPConnectionPool, - ResponseDone, - ResponseFailed, -) +from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import Response as TxResponse +from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse +from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: @@ -46,28 +48,30 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler): + def __init__(self, settings: BaseSettings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor - self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) self._pool._factory.noisy = False - self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") - self._disconnect_timeout = 1 + self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( + settings, crawler + ) + self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") + self._disconnect_timeout: int = 1 @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -79,10 +83,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self): + def close(self) -> Deferred: from twisted.internet import reactor - d = self._pool.closeCachedConnections() + d: Deferred = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -93,7 +97,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result): + def cancel_delayed_call(result: Any) -> Any: if delayed_call.active(): delayed_call.cancel() return result @@ -123,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__( self, - reactor, - host, - port, - proxyConf, - contextFactory, - timeout=30, - bindAddress=None, + reactor: ReactorBase, + host: str, + port: int, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: IPolicyForHTTPS, + timeout: float = 30, + bindAddress: Optional[Tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred = defer.Deferred() - self._tunneledHost = host - self._tunneledPort = port - self._contextFactory = contextFactory - self._connectBuffer = bytearray() + self._tunnelReadyDeferred: Deferred = Deferred() + self._tunneledHost: str = host + self._tunneledPort: int = port + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectBuffer: bytearray = bytearray() - def requestTunnel(self, protocol): + def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" + assert protocol.transport tunnelReq = tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived - protocol.dataReceived = self.processProxyResponse + protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign] self._protocol = protocol return protocol - def processProxyResponse(self, rcvd_bytes): + def processProxyResponse(self, data: bytes) -> None: """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ - self._connectBuffer += rcvd_bytes + assert self._protocol.transport + self._connectBuffer += data # make sure that enough (all) bytes are consumed # and that we've got all HTTP headers (ending with a blank line) # from the proxy so that we don't send those bytes to the TLS layer @@ -163,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # see https://github.com/scrapy/scrapy/issues/2491 if b"\r\n\r\n" not in self._connectBuffer: return - self._protocol.dataReceived = self._protocolDataReceived + self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( + sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] self._tunneledHost, self._tunneledPort ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + extra: Any if respm: extra = { "status": int(respm.group("status")), "reason": respm.group("reason").strip(), } else: - extra = rcvd_bytes[: self._truncatedLength] + extra = data[: self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError( "Could not open CONNECT tunnel with proxy " @@ -187,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ) ) - def connectFailed(self, reason): + def connectFailed(self, reason: Failure) -> None: """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory): + def connect(self, protocolFactory: Factory) -> Deferred: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -199,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def tunnel_request_data( + host: str, port: int, proxy_auth_header: Optional[bytes] = None +) -> bytes: r""" Return binary content of a CONNECT request. @@ -230,18 +239,20 @@ class TunnelingAgent(Agent): def __init__( self, - reactor, - proxyConf, - contextFactory=None, - connectTimeout=None, - bindAddress=None, - pool=None, + reactor: ReactorBase, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): + # TODO make this arg required instead + assert contextFactory is not None super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf = proxyConf - self._contextFactory = contextFactory + self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri): + def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: return TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, @@ -253,8 +264,15 @@ class TunnelingAgent(Agent): ) def _requestWithEndpoint( - self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath - ): + self, + key: Any, + endpoint: TCP4ClientEndpoint, + method: bytes, + parsedURI: bytes, + headers: Optional[TxHeaders], + bodyProducer: Optional[IBodyProducer], + requestPath: bytes, + ) -> Deferred: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -272,7 +290,12 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__( - self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + self, + reactor: ReactorBase, + proxyURI: bytes, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): super().__init__( reactor=reactor, @@ -280,9 +303,15 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI: URI = URI.fromBytes(proxyURI) - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[TxHeaders] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> Deferred: """ Issue a new request via the configured proxy. """ @@ -306,26 +335,29 @@ class ScrapyAgent: def __init__( self, - contextFactory=None, - connectTimeout=10, - bindAddress=None, - pool=None, - maxsize=0, - warnsize=0, - fail_on_dataloss=True, - crawler=None, + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: float = 10, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, + maxsize: int = 0, + warnsize: int = 0, + fail_on_dataloss: bool = True, + crawler: Optional[Crawler] = None, ): - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._txresponse = None - self._crawler = crawler + # TODO make these args required instead + assert contextFactory is not None + assert crawler is not None + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectTimeout: float = connectTimeout + self._bindAddress: Optional[bytes] = bindAddress + self._pool: Optional[HTTPConnectionPool] = pool + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._txresponse: Optional[TxResponse] = None + self._crawler: Crawler = crawler - def _get_agent(self, request, timeout): + def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress @@ -333,10 +365,10 @@ class ScrapyAgent: if proxy: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - proxyHost = to_unicode(proxyHost) + proxyHost_str = to_unicode(proxyHost) if scheme == b"https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost, proxyPort, proxyAuth) + proxyConf = (proxyHost_str, proxyPort, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -346,7 +378,9 @@ class ScrapyAgent: pool=self._pool, ) proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) + proxyURI = urlunparse( + (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") + ) return self._ProxyAgent( reactor=reactor, proxyURI=to_bytes(proxyURI, encoding="ascii"), @@ -363,7 +397,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request): + def download_request(self, request: Request) -> Deferred: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -380,7 +414,7 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d = agent.request( + d: Deferred = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency @@ -393,7 +427,9 @@ class ScrapyAgent: d.addBoth(self._cb_timeout, request, url, timeout) return d - def _cb_timeout(self, result, request, url, timeout): + def _cb_timeout( + self, result: Any, request: Request, url: str, timeout: float + ) -> Any: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -404,19 +440,21 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result, request, start_time): + def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: request.meta["download_latency"] = time() - start_time return result @staticmethod - def _headers_from_twisted_response(response): + def _headers_from_twisted_response(response: TxResponse) -> Headers: headers = Headers() if response.length != UNKNOWN_LENGTH: headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers - def _cb_bodyready(self, txresponse, request): + def _cb_bodyready( + self, txresponse: TxResponse, request: Request + ) -> Union[Dict[str, Any], Deferred]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -472,7 +510,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) txresponse._transport.loseConnection() - raise defer.CancelledError(warning_msg % warning_args) + raise CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning( @@ -481,11 +519,11 @@ class ScrapyAgent: {"size": expected_size, "warnsize": warnsize, "request": request}, ) - def _cancel(_): + def _cancel(_: Any) -> None: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d = defer.Deferred(_cancel) + d: Deferred = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -503,7 +541,9 @@ class ScrapyAgent: return d - def _cb_bodydone(self, result, request, url): + def _cb_bodydone( + self, result: Dict[str, Any], request: Request, url: str + ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -523,53 +563,57 @@ class ScrapyAgent: ) if result.get("failure"): result["failure"].value.response = response - return result["failure"] + return cast(Failure, result["failure"]) return response @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): + def __init__(self, body: bytes): self.body = body self.length = len(body) - def startProducing(self, consumer): + def startProducing(self, consumer: IConsumer) -> Deferred: consumer.write(self.body) - return defer.succeed(None) + return succeed(None) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass -class _ResponseReader(protocol.Protocol): +class _ResponseReader(Protocol): def __init__( self, - finished, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - crawler, + finished: Deferred, + txresponse: TxResponse, + request: Request, + maxsize: int, + warnsize: int, + fail_on_dataloss: bool, + crawler: Crawler, ): - self._finished = finished - self._txresponse = txresponse - self._request = request - self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._fail_on_dataloss_warned = False - self._reached_warnsize = False - self._bytes_received = 0 - self._certificate = None - self._ip_address = None - self._crawler = crawler + self._finished: Deferred = finished + self._txresponse: TxResponse = txresponse + self._request: Request = request + self._bodybuf: BytesIO = BytesIO() + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._fail_on_dataloss_warned: bool = False + self._reached_warnsize: bool = False + self._bytes_received: int = 0 + self._certificate: Optional[ssl.Certificate] = None + self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( + None + ) + self._crawler: Crawler = crawler - def _finish_response(self, flags=None, failure=None): + def _finish_response( + self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + ) -> None: self._finished.callback( { "txresponse": self._txresponse, @@ -581,7 +625,8 @@ class _ResponseReader(protocol.Protocol): } ) - def connectionMade(self): + def connectionMade(self) -> None: + assert self.transport if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( @@ -593,11 +638,12 @@ class _ResponseReader(protocol.Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes): + def dataReceived(self, bodyBytes: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return + assert self.transport self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) @@ -644,7 +690,7 @@ class _ResponseReader(protocol.Protocol): {"warnsize": self._warnsize, "request": self._request}, ) - def connectionLost(self, reason): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self._finished.called: return diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index c88dd2cdc..9a0811a50 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,11 +2,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type +from twisted.internet.defer import Deferred + from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -74,7 +75,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname