From ec4d40702227f5486c184de9439da4896121a33b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 17:36:16 +0500 Subject: [PATCH] Full typing for smaller download handlers. --- scrapy/core/downloader/handlers/file.py | 4 +++- scrapy/core/downloader/handlers/http11.py | 6 ++--- scrapy/core/downloader/handlers/http2.py | 3 ++- scrapy/core/downloader/handlers/s3.py | 27 ++++++++++++++--------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 4824167da..17dd7483b 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,6 +2,8 @@ from pathlib import Path from w3lib.url import file_uri_to_path +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -10,7 +12,7 @@ class FileDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: filepath = file_uri_to_path(request.url) body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 52561f7e6..2e7ea559a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -27,11 +27,11 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from zope.interface import implementer -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import StopDownload -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -67,7 +67,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index efe45c459..16fc1e3ae 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -8,6 +8,7 @@ from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI +from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -54,7 +55,7 @@ class ScrapyH2Agent: def __init__( self, - context_factory, + context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, bind_address: Optional[bytes] = None, diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 99fbb49ce..c88dd2cdc 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional, Type +from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler @@ -16,14 +20,14 @@ if TYPE_CHECKING: class S3DownloadHandler: def __init__( self, - settings, + settings: BaseSettings, *, - crawler=None, - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, - **kw, + crawler: Crawler, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + **kw: Any, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -51,6 +55,8 @@ class S3DownloadHandler: if kw: raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: + assert aws_access_key_id is not None + assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] self._signer = SignerCls( botocore.credentials.Credentials( @@ -65,10 +71,10 @@ class S3DownloadHandler: self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs) -> Self: + def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname @@ -85,6 +91,7 @@ class S3DownloadHandler: headers=request.headers.to_unicode_dict(), data=request.body, ) + assert self._signer self._signer.add_auth(awsrequest) request = request.replace(url=url, headers=awsrequest.headers.items()) return self._download_http(request, spider)