diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 16639356e..67b22dbc5 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser): curl_parser = CurlParser() curl_parser.add_argument('url') curl_parser.add_argument('-H', '--header', dest='headers', action='append') -curl_parser.add_argument('-X', '--request', dest='method', default='get') -curl_parser.add_argument('-d', '--data', dest='data') +curl_parser.add_argument('-X', '--request', dest='method') +curl_parser.add_argument('-d', '--data', '--data-raw', dest='data') curl_parser.add_argument('-u', '--user', dest='auth') @@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): if not parsed_url.scheme: url = 'http://' + url - result = {'method': parsed_args.method.upper(), 'url': url} + method = parsed_args.method or 'GET' + + result = {'method': method.upper(), 'url': url} headers = [] cookies = {} @@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result['cookies'] = cookies if parsed_args.data: result['body'] = parsed_args.data + if not parsed_args.method: + # if the "data" is specified but the "method" is not specified, + # the default method is 'POST' + result['method'] = 'POST' return result diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 50e1bfd5f..299a51efe 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase): } self._test_command(curl_command, expected_result) + def test_post_data_raw(self): + curl_command = ( + "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena" + "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc" + "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'" + ) + expected_result = { + "method": "POST", + "url": "https://www.example.org/", + "body": ( + "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3" + "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking" + "Functions=%5B%5D") + } + self._test_command(curl_command, expected_result) + + def test_explicit_get_with_data(self): + curl_command = 'curl httpbin.org/anything -X GET --data asdf' + expected_result = { + "method": "GET", + "url": "http://httpbin.org/anything", + "body": "asdf" + } + self._test_command(curl_command, expected_result) + def test_patch(self): curl_command = ( 'curl "https://example.com/api/fake" -u "username:password" -H "Ac'