Full typing for smaller download handlers.

scrapy/core/downloader/handlers/file.py

@@ -2,6 +2,8 @@ from pathlib import Path
 
 from w3lib.url import file_uri_to_path
 
+from scrapy import Request, Spider
+from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
@@ -10,7 +12,7 @@ class FileDownloadHandler:
     lazy = False
 
     @defers
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         filepath = file_uri_to_path(request.url)
         body = Path(filepath).read_bytes()
         respcls = responsetypes.from_args(filename=filepath, body=body)

scrapy/core/downloader/handlers/http11.py

@@ -27,11 +27,11 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from zope.interface import implementer
 
-from scrapy import signals
+from scrapy import Request, Spider, signals
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import StopDownload
-from scrapy.http import Headers
+from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.python import to_bytes, to_unicode
 
@@ -67,7 +67,7 @@ class HTTP11DownloadHandler:
     def from_crawler(cls, crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,

scrapy/core/downloader/handlers/http2.py

@@ -8,6 +8,7 @@ from twisted.internet.base import DelayedCall
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
+from twisted.web.iweb import IPolicyForHTTPS
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
@@ -54,7 +55,7 @@ class ScrapyH2Agent:
 
     def __init__(
         self,
-        context_factory,
+        context_factory: IPolicyForHTTPS,
         pool: H2ConnectionPool,
         connect_timeout: int = 10,
         bind_address: Optional[bytes] = None,

scrapy/core/downloader/handlers/s3.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, Type
 
+from scrapy import Request, Spider
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+from scrapy.settings import BaseSettings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler
@@ -16,14 +20,14 @@ if TYPE_CHECKING:
 class S3DownloadHandler:
     def __init__(
         self,
-        settings,
+        settings: BaseSettings,
         *,
-        crawler=None,
-        aws_access_key_id=None,
-        aws_secret_access_key=None,
-        aws_session_token=None,
-        httpdownloadhandler=HTTPDownloadHandler,
-        **kw,
+        crawler: Crawler,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        aws_session_token: Optional[str] = None,
+        httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler,
+        **kw: Any,
     ):
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
@@ -51,6 +55,8 @@ class S3DownloadHandler:
         if kw:
             raise TypeError(f"Unexpected keyword arguments: {kw}")
         if not self.anon:
+            assert aws_access_key_id is not None
+            assert aws_secret_access_key is not None
             SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
             self._signer = SignerCls(
                 botocore.credentials.Credentials(
@@ -65,10 +71,10 @@ class S3DownloadHandler:
         self._download_http = _http_handler.download_request
 
     @classmethod
-    def from_crawler(cls, crawler, **kwargs) -> Self:
+    def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
         return cls(crawler.settings, crawler=crawler, **kwargs)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         p = urlparse_cached(request)
         scheme = "https" if request.meta.get("is_secure") else "http"
         bucket = p.hostname
@@ -85,6 +91,7 @@ class S3DownloadHandler:
                 headers=request.headers.to_unicode_dict(),
                 data=request.body,
             )
+            assert self._signer
             self._signer.add_auth(awsrequest)
             request = request.replace(url=url, headers=awsrequest.headers.items())
         return self._download_http(request, spider)