From 78ee47c659b1984aa3b1def594f6f49debdfc845 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 27 Jun 2026 13:50:54 +0500 Subject: [PATCH] Extract _to_verbose_cookies(). --- scrapy/downloadermiddlewares/cookies.py | 8 ++--- scrapy/utils/request.py | 35 ++++++++++++---------- tests/test_downloadermiddleware_cookies.py | 11 +++++-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 1186947b4..34d071fd5 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -12,7 +12,7 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -from scrapy.utils.request import _decode_cookie +from scrapy.utils.request import _decode_cookie, _to_verbose_cookies if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -159,11 +159,7 @@ class CookiesMiddleware: """ if not request.cookies: return () - cookies: Iterable[VerboseCookie] - if isinstance(request.cookies, dict): - cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies + cookies: Iterable[VerboseCookie] = _to_verbose_cookies(request.cookies) for cookie in cookies: cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index d756ab7b3..b2fd9a834 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -26,8 +26,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler - from scrapy.http.request import VerboseCookie - + from scrapy.http.request import CookiesT, VerboseCookie logger = logging.getLogger(__name__) @@ -184,6 +183,16 @@ def _get_method(obj: Any, name: Any) -> Any: raise ValueError(f"Method {name!r} not found in: {obj}") from None +def _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]: + """Return a list of verbose cookies from ``request.cookies``. + + The list of dicts form is returned as is, the dict one is converted first. + """ + if isinstance(cookies, dict): + return [{"name": k, "value": v} for k, v in cookies.items()] + return cookies + + def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None: """Return a dict with non-flag verbose cookie values converted to strings. @@ -231,20 +240,14 @@ def request_to_curl(request: Request) -> str: ) url = request.url - cookies = "" - if request.cookies: - cookie_list: list[VerboseCookie] = ( - [{"name": k, "value": v} for k, v in request.cookies.items()] - if isinstance(request.cookies, dict) - else request.cookies - ) - pairs = [ - f"{decoded['name']}={decoded['value']}" - for c in cookie_list - if (decoded := _decode_cookie(c, request)) is not None - ] - if pairs: - cookies = f"--cookie '{'; '.join(pairs)}'" + + cookie_list: list[VerboseCookie] = _to_verbose_cookies(request.cookies) + pairs = [ + f"{decoded['name']}={decoded['value']}" + for c in cookie_list + if (decoded := _decode_cookie(c, request)) is not None + ] + cookies = f"--cookie '{'; '.join(pairs)}'" if pairs else "" curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() return " ".join(curl_cmd.split()) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 88460836c..fcfc8fdb5 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -9,6 +9,7 @@ from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.utils.python import to_bytes +from scrapy.utils.request import _to_verbose_cookies from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler @@ -46,9 +47,13 @@ def _cookies_to_set_cookie_list(cookies): ``Set-Cookie`` header.""" if not cookies: return [] - if isinstance(cookies, dict): - cookies = ({"name": k, "value": v} for k, v in cookies.items()) - return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) + return filter( + None, + ( + _cookie_to_set_cookie_value(cookie) + for cookie in _to_verbose_cookies(cookies) + ), + ) class TestCookiesMiddleware: