Extract _decode_cookie().

scrapy/downloadermiddlewares/cookies.py

@@ -12,6 +12,7 @@ from scrapy.http.cookies import CookieJar
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
+from scrapy.utils.request import _decode_cookie
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Sequence
@@ -134,29 +135,10 @@ class CookiesMiddleware:
         Given a dict consisting of cookie components, return its string representation.
         Decode from bytes if necessary.
         """
-        decoded = {}
+        decoded = _decode_cookie(cookie, request)
+        if decoded is None:
+            return None
         flags = set()
-        for key in ("name", "value", "path", "domain"):
-            value = cookie.get(key)
-            if value is None:
-                if key in {"name", "value"}:
-                    msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
-                    logger.warning(msg)
-                    return None
-                continue
-            if isinstance(value, (bool, float, int, str)):
-                decoded[key] = str(value)
-            else:
-                assert isinstance(value, bytes)
-                try:
-                    decoded[key] = value.decode("utf8")
-                except UnicodeDecodeError:
-                    logger.warning(
-                        "Non UTF-8 encoded cookie found in request %s: %s",
-                        request,
-                        cookie,
-                    )
-                    decoded[key] = value.decode("latin1", errors="replace")
         for flag in ("secure",):
             value = cookie.get(flag, _UNSET)
             if value is _UNSET or not value:

scrapy/utils/request.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 from typing import TYPE_CHECKING, Any, Protocol
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
@@ -25,6 +26,10 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import VerboseCookie
+
+
+logger = logging.getLogger(__name__)
 
 
 _fingerprint_cache: WeakKeyDictionary[
@@ -179,17 +184,42 @@ def _get_method(obj: Any, name: Any) -> Any:
         raise ValueError(f"Method {name!r} not found in: {obj}") from None
 
 
-def _cookie_value_to_unicode(value: str | bytes | float) -> str:
-    if isinstance(value, bytes):
-        return value.decode()
-    return str(value)
+def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None:
+    """Return a dict with non-flag verbose cookie values converted to strings.
+
+    ``name``, ``value``, ``path``, ``domain`` are included, ``secure`` isn't.
+    """
+
+    decoded = {}
+    for key in ("name", "value", "path", "domain"):
+        value = cookie.get(key)
+        if value is None:
+            if key in {"name", "value"}:
+                msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+                logger.warning(msg)
+                return None
+            continue
+        if isinstance(value, (bool, float, int, str)):
+            decoded[key] = str(value)
+        else:
+            assert isinstance(value, bytes)
+            try:
+                decoded[key] = value.decode("utf8")
+            except UnicodeDecodeError:
+                logger.warning(
+                    "Non UTF-8 encoded cookie found in request %s: %s",
+                    request,
+                    cookie,
+                )
+                decoded[key] = value.decode("latin1", errors="replace")
+    return decoded
 
 
 def request_to_curl(request: Request) -> str:
     """
     Converts a :class:`~scrapy.Request` object to a curl command.
 
-    :param :class:`~scrapy.Request`: Request object to be converted
+    :param request: Request object to be converted
     :return: string containing the curl command
     """
     method = request.method
@@ -203,19 +233,18 @@ def request_to_curl(request: Request) -> str:
     url = request.url
     cookies = ""
     if request.cookies:
-        if isinstance(request.cookies, dict):
-            cookie = "; ".join(
-                f"{_cookie_value_to_unicode(k)}={_cookie_value_to_unicode(v)}"
-                for k, v in request.cookies.items()
-            )
-            cookies = f"--cookie '{cookie}'"
-        elif isinstance(request.cookies, list):
-            cookie = "; ".join(
-                f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}"
-                for c in request.cookies
-                if "name" in c and "value" in c
-            )
-            cookies = f"--cookie '{cookie}'"
+        cookie_list: list[VerboseCookie] = (
+            [{"name": k, "value": v} for k, v in request.cookies.items()]
+            if isinstance(request.cookies, dict)
+            else request.cookies
+        )
+        pairs = [
+            f"{decoded['name']}={decoded['value']}"
+            for c in cookie_list
+            if (decoded := _decode_cookie(c, request)) is not None
+        ]
+        if pairs:
+            cookies = f"--cookie '{'; '.join(pairs)}'"
 
     curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()
     return " ".join(curl_cmd.split())

tests/test_downloadermiddleware_cookies.py

@@ -410,7 +410,7 @@ class TestCookiesMiddleware:
         Invalid cookies are logged as warnings and discarded
         """
         with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
+            "scrapy.utils.request",
             propagate=False,
             level=logging.INFO,
         ) as lc:
@@ -425,19 +425,19 @@ class TestCookiesMiddleware:
             assert self.mw.process_request(req3) is None
             lc.check(
                 (
-                    "scrapy.downloadermiddlewares.cookies",
+                    "scrapy.utils.request",
                     "WARNING",
                     "Invalid cookie found in request <GET http://example.org/1>:"
                     " {'value': 'bar', 'secure': False} ('name' is missing)",
                 ),
                 (
-                    "scrapy.downloadermiddlewares.cookies",
+                    "scrapy.utils.request",
                     "WARNING",
                     "Invalid cookie found in request <GET http://example.org/2>:"
                     " {'name': 'foo', 'secure': False} ('value' is missing)",
                 ),
                 (
-                    "scrapy.downloadermiddlewares.cookies",
+                    "scrapy.utils.request",
                     "WARNING",
                     "Invalid cookie found in request <GET http://example.org/3>:"
                     " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",