diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a2243ae2e..f5dbbd64e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -7,6 +7,14 @@ from urllib.parse import urlparse from w3lib.http import basic_auth_header +class DataAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + value = str(values) + if value.startswith("$"): + value = value[1:] + setattr(namespace, self.dest, value) + + class CurlParser(argparse.ArgumentParser): def error(self, message): error_msg = f"There was an error parsing the curl command: {message}" @@ -17,7 +25,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") -curl_parser.add_argument("-d", "--data", "--data-raw", dest="data") +curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fd4612eba..1816db29b 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -154,6 +154,15 @@ class CurlToRequestKwargsTest(unittest.TestCase): } self._test_command(curl_command, expected_result) + def test_post_data_raw_with_string_prefix(self): + curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'" + expected_result = { + "method": "POST", + "url": "https://www.example.org/", + "body": '{"$filters":"Filter!"}', + } + self._test_command(curl_command, expected_result) + def test_explicit_get_with_data(self): curl_command = "curl httpbin.org/anything -X GET --data asdf" expected_result = {