Use a TypedDict for the verbose cookie form.

scrapy/downloadermiddlewares/cookies.py

@@ -3,16 +3,7 @@ from __future__ import annotations
 import logging
 from collections import defaultdict
 from http.cookiejar import Cookie
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    DefaultDict,
-    Dict,
-    Iterable,
-    Optional,
-    Sequence,
-    Union,
-)
+from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
 
 from tldextract import TLDExtract
 
@@ -21,6 +12,7 @@ from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
+from scrapy.http.request import VerboseCookie
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
@@ -128,7 +120,7 @@ class CookiesMiddleware:
                 msg = f"Received cookies from: {response}\n{cookies}"
                 logger.debug(msg, extra={"spider": spider})
 
-    def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
+    def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]:
         """
         Given a dict consisting of cookie components, return its string representation.
         Decode from bytes if necessary.
@@ -142,18 +134,19 @@ class CookiesMiddleware:
                     logger.warning(msg)
                     return None
                 continue
-            if isinstance(cookie[key], (bool, float, int, str)):
-                decoded[key] = str(cookie[key])
+            # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
+            if isinstance(cookie[key], (bool, float, int, str)):  # type: ignore[literal-required]
+                decoded[key] = str(cookie[key])  # type: ignore[literal-required]
             else:
                 try:
-                    decoded[key] = cookie[key].decode("utf8")
+                    decoded[key] = cookie[key].decode("utf8")  # type: ignore[literal-required]
                 except UnicodeDecodeError:
                     logger.warning(
                         "Non UTF-8 encoded cookie found in request %s: %s",
                         request,
                         cookie,
                     )
-                    decoded[key] = cookie[key].decode("latin1", errors="replace")
+                    decoded[key] = cookie[key].decode("latin1", errors="replace")  # type: ignore[literal-required]
         for flag in ("secure",):
             value = cookie.get(flag, _UNSET)
             if value is _UNSET or not value:
@@ -174,7 +167,7 @@ class CookiesMiddleware:
         """
         if not request.cookies:
             return []
-        cookies: Iterable[Dict[str, Any]]
+        cookies: Iterable[VerboseCookie]
         if isinstance(request.cookies, dict):
             cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
         else:

scrapy/http/request/__init__.py

@@ -20,6 +20,7 @@ from typing import (
     NoReturn,
     Optional,
     Tuple,
+    TypedDict,
     Union,
     cast,
 )
@@ -34,8 +35,19 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    # typing.NotRequired and typing.Self require Python 3.11
+    from typing_extensions import NotRequired, Self
+
+
+class VerboseCookie(TypedDict):
+    name: str
+    value: str
+    domain: NotRequired[str]
+    path: NotRequired[str]
+    secure: NotRequired[bool]
+
+
+CookiesT = Union[Dict[str, str], List[VerboseCookie]]
 
 
 def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
@@ -97,7 +109,7 @@ class Request(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: str = "utf-8",
         priority: int = 0,
@@ -123,7 +135,7 @@ class Request(object_ref):
         self.callback: Optional[Callable] = callback
         self.errback: Optional[Callable] = errback
 
-        self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {}
+        self.cookies: CookiesT = cookies or {}
         self.headers: Headers = Headers(headers or {}, encoding=encoding)
         self.dont_filter: bool = dont_filter
 

scrapy/http/response/__init__.py

@@ -29,7 +29,7 @@ from twisted.internet.ssl import Certificate
 
 from scrapy.exceptions import NotSupported
 from scrapy.http.headers import Headers
-from scrapy.http.request import Request
+from scrapy.http.request import CookiesT, Request
 from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
@@ -181,7 +181,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
@@ -234,7 +234,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,

scrapy/http/response/text.py

@@ -36,7 +36,7 @@ from w3lib.encoding import (
 )
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.http import Request
+from scrapy.http.request import CookiesT, Request
 from scrapy.http.response import Response
 from scrapy.link import Link
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -183,7 +183,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,
@@ -236,7 +236,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,