diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 409ca2e52..6d8be991d 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -327,3 +327,34 @@ def _get_method(obj, name): return getattr(obj, name) except AttributeError: raise ValueError(f"Method {name!r} not found in: {obj}") + + +def request_to_curl(request: Request) -> str: + """ + Converts a :class:`~scrapy.Request` object to a curl command. + + :param :class:`~scrapy.Request`: Request object to be converted + :return: string containing the curl command + """ + method = request.method + + data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else "" + + headers = " ".join( + f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items() + ) + + url = request.url + cookies = "" + if request.cookies: + if isinstance(request.cookies, dict): + cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items()) + cookies = f"--cookie '{cookie}'" + elif isinstance(request.cookies, list): + cookie = "; ".join( + f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + ) + cookies = f"--cookie '{cookie}'" + + curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() + return " ".join(curl_cmd.split()) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 6ca272de1..e6d1abe3f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,3 +1,4 @@ +import json import unittest import warnings from hashlib import sha1 @@ -18,6 +19,7 @@ from scrapy.utils.request import ( request_authenticate, request_fingerprint, request_httprepr, + request_to_curl, ) from scrapy.utils.test import get_crawler @@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertEqual(fingerprint, settings["FINGERPRINT"]) +class RequestToCurlTest(unittest.TestCase): + def _test_request(self, request_object, expected_curl_command): + curl_command = request_to_curl(request_object) + self.assertEqual(curl_command, expected_curl_command) + + def test_get(self): + request_object = Request("https://www.example.com") + expected_curl_command = "curl -X GET https://www.example.com" + self._test_request(request_object, expected_curl_command) + + def test_post(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + self._test_request(request_object, expected_curl_command) + + def test_headers(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + ' --data-raw \'{"foo": "bar"}\'' + " -H 'Content-Type: application/json' -H 'Accept: application/json'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_dict(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies={"foo": "bar"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_list(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[{"foo": "bar"}], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + if __name__ == "__main__": unittest.main()