From 859a77ee4243f17f338072e45785383f12516308 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 22:23:26 +0500 Subject: [PATCH] Use a TypedDict for the verbose cookie form. --- scrapy/downloadermiddlewares/cookies.py | 25 +++++++++---------------- scrapy/http/request/__init__.py | 20 ++++++++++++++++---- scrapy/http/response/__init__.py | 6 +++--- scrapy/http/response/text.py | 6 +++--- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6ada3b474..73c2c57fe 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -3,16 +3,7 @@ from __future__ import annotations import logging from collections import defaultdict from http.cookiejar import Cookie -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Dict, - Iterable, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract @@ -21,6 +12,7 @@ from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -128,7 +120,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -142,18 +134,19 @@ class CookiesMiddleware: logger.warning(msg) return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 + if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] + decoded[key] = str(cookie[key]) # type: ignore[literal-required] else: try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") + decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: @@ -174,7 +167,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - cookies: Iterable[Dict[str, Any]] + cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index dfb1dca89..96d0dc515 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -20,6 +20,7 @@ from typing import ( NoReturn, Optional, Tuple, + TypedDict, Union, cast, ) @@ -34,8 +35,19 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self + + +class VerboseCookie(TypedDict): + name: str + value: str + domain: NotRequired[str] + path: NotRequired[str] + secure: NotRequired[bool] + + +CookiesT = Union[Dict[str, str], List[VerboseCookie]] def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -97,7 +109,7 @@ class Request(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, @@ -123,7 +135,7 @@ class Request(object_ref): self.callback: Optional[Callable] = callback self.errback: Optional[Callable] = errback - self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {} + self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 14618e5e7..166c4de97 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -29,7 +29,7 @@ from twisted.internet.ssl import Certificate from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import Request +from scrapy.http.request import CookiesT, Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -181,7 +181,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, @@ -234,7 +234,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a83279ac8..44c36b682 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -36,7 +36,7 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http import Request +from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -183,7 +183,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, @@ -236,7 +236,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0,