From 12a5ea1a62c21c1a83916a0e8e1859505ada5be7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 27 Jun 2026 13:31:09 +0500 Subject: [PATCH] Extract _decode_cookie(). --- scrapy/downloadermiddlewares/cookies.py | 26 ++------- scrapy/utils/request.py | 65 ++++++++++++++++------ tests/test_downloadermiddleware_cookies.py | 8 +-- 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index cd8c2abca..1186947b4 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -12,6 +12,7 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +from scrapy.utils.request import _decode_cookie if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -134,29 +135,10 @@ class CookiesMiddleware: Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. """ - decoded = {} + decoded = _decode_cookie(cookie, request) + if decoded is None: + return None flags = set() - for key in ("name", "value", "path", "domain"): - value = cookie.get(key) - if value is None: - if key in {"name", "value"}: - msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" - logger.warning(msg) - return None - continue - if isinstance(value, (bool, float, int, str)): - decoded[key] = str(value) - else: - assert isinstance(value, bytes) - try: - decoded[key] = value.decode("utf8") - except UnicodeDecodeError: - logger.warning( - "Non UTF-8 encoded cookie found in request %s: %s", - request, - cookie, - ) - decoded[key] = value.decode("latin1", errors="replace") for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db64407bf..36e71a821 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -7,6 +7,7 @@ from __future__ import annotations import hashlib import json +import logging from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -25,6 +26,10 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + + +logger = logging.getLogger(__name__) _fingerprint_cache: WeakKeyDictionary[ @@ -179,17 +184,42 @@ def _get_method(obj: Any, name: Any) -> Any: raise ValueError(f"Method {name!r} not found in: {obj}") from None -def _cookie_value_to_unicode(value: str | bytes | float) -> str: - if isinstance(value, bytes): - return value.decode() - return str(value) +def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None: + """Return a dict with non-flag verbose cookie values converted to strings. + + ``name``, ``value``, ``path``, ``domain`` are included, ``secure`` isn't. + """ + + decoded = {} + for key in ("name", "value", "path", "domain"): + value = cookie.get(key) + if value is None: + if key in {"name", "value"}: + msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" + logger.warning(msg) + return None + continue + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) + else: + assert isinstance(value, bytes) + try: + decoded[key] = value.decode("utf8") + except UnicodeDecodeError: + logger.warning( + "Non UTF-8 encoded cookie found in request %s: %s", + request, + cookie, + ) + decoded[key] = value.decode("latin1", errors="replace") + return decoded def request_to_curl(request: Request) -> str: """ Converts a :class:`~scrapy.Request` object to a curl command. - :param :class:`~scrapy.Request`: Request object to be converted + :param request: Request object to be converted :return: string containing the curl command """ method = request.method @@ -203,19 +233,18 @@ def request_to_curl(request: Request) -> str: url = request.url cookies = "" if request.cookies: - if isinstance(request.cookies, dict): - cookie = "; ".join( - f"{_cookie_value_to_unicode(k)}={_cookie_value_to_unicode(v)}" - for k, v in request.cookies.items() - ) - cookies = f"--cookie '{cookie}'" - elif isinstance(request.cookies, list): - cookie = "; ".join( - f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}" - for c in request.cookies - if "name" in c and "value" in c - ) - cookies = f"--cookie '{cookie}'" + cookie_list: list[VerboseCookie] = ( + [{"name": k, "value": v} for k, v in request.cookies.items()] + if isinstance(request.cookies, dict) + else request.cookies + ) + pairs = [ + f"{decoded['name']}={decoded['value']}" + for c in cookie_list + if (decoded := _decode_cookie(c, request)) is not None + ] + if pairs: + cookies = f"--cookie '{'; '.join(pairs)}'" curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() return " ".join(curl_cmd.split()) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index f79591020..88460836c 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -410,7 +410,7 @@ class TestCookiesMiddleware: Invalid cookies are logged as warnings and discarded """ with LogCapture( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", propagate=False, level=logging.INFO, ) as lc: @@ -425,19 +425,19 @@ class TestCookiesMiddleware: assert self.mw.process_request(req3) is None lc.check( ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",