Implement a request_to_curl function (#5892)

scrapy/utils/request.py

@@ -327,3 +327,34 @@ def _get_method(obj, name):
         return getattr(obj, name)
     except AttributeError:
         raise ValueError(f"Method {name!r} not found in: {obj}")
+
+
+def request_to_curl(request: Request) -> str:
+    """
+    Converts a :class:`~scrapy.Request` object to a curl command.
+
+    :param :class:`~scrapy.Request`: Request object to be converted
+    :return: string containing the curl command
+    """
+    method = request.method
+
+    data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else ""
+
+    headers = " ".join(
+        f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items()
+    )
+
+    url = request.url
+    cookies = ""
+    if request.cookies:
+        if isinstance(request.cookies, dict):
+            cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items())
+            cookies = f"--cookie '{cookie}'"
+        elif isinstance(request.cookies, list):
+            cookie = "; ".join(
+                f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies
+            )
+            cookies = f"--cookie '{cookie}'"
+
+    curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()
+    return " ".join(curl_cmd.split())

tests/test_utils_request.py

@@ -1,3 +1,4 @@
+import json
 import unittest
 import warnings
 from hashlib import sha1
@@ -18,6 +19,7 @@ from scrapy.utils.request import (
     request_authenticate,
     request_fingerprint,
     request_httprepr,
+    request_to_curl,
 )
 from scrapy.utils.test import get_crawler
 
@@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
         self.assertEqual(fingerprint, settings["FINGERPRINT"])
 
 
+class RequestToCurlTest(unittest.TestCase):
+    def _test_request(self, request_object, expected_curl_command):
+        curl_command = request_to_curl(request_object)
+        self.assertEqual(curl_command, expected_curl_command)
+
+    def test_get(self):
+        request_object = Request("https://www.example.com")
+        expected_curl_command = "curl -X GET https://www.example.com"
+        self._test_request(request_object, expected_curl_command)
+
+    def test_post(self):
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
+        )
+        self._test_request(request_object, expected_curl_command)
+
+    def test_headers(self):
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            headers={"Content-Type": "application/json", "Accept": "application/json"},
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            "curl -X POST https://www.httpbin.org/post"
+            ' --data-raw \'{"foo": "bar"}\''
+            " -H 'Content-Type: application/json' -H 'Accept: application/json'"
+        )
+        self._test_request(request_object, expected_curl_command)
+
+    def test_cookies_dict(self):
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            cookies={"foo": "bar"},
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            "curl -X POST https://www.httpbin.org/post"
+            " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+        )
+        self._test_request(request_object, expected_curl_command)
+
+    def test_cookies_list(self):
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            cookies=[{"foo": "bar"}],
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            "curl -X POST https://www.httpbin.org/post"
+            " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+        )
+        self._test_request(request_object, expected_curl_command)
+
+
 if __name__ == "__main__":
     unittest.main()