mirror of https://github.com/scrapy/scrapy.git
Merge repeated curl -d/--data options in Request.from_curl()
curl merges repeated -d/--data/--data-raw options into a single request
body joined with "&" (e.g. `curl -d a=1 -d b=2` sends `a=1&b=2`). scrapy's
DataAction kept only the last value, silently dropping the earlier ones, so
`Request.from_curl("curl -d a=1 -d b=2 ...")` produced a body of just `b=2`.
Accumulate the values with "&" to match curl. -H/--header and -b/--cookie
already accumulate via action="append"; -d was the only repeatable data flag
that didn't.
This commit is contained in:
parent
c9446931a8
commit
8ec0347b27
|
|
@ -23,6 +23,11 @@ class DataAction(argparse.Action):
|
|||
) -> None:
|
||||
value = str(values)
|
||||
value = value.removeprefix("$")
|
||||
# curl merges repeated -d/--data/--data-raw options into a single body
|
||||
# joined with "&"; mirror that instead of keeping only the last one.
|
||||
previous = getattr(namespace, self.dest, None)
|
||||
if previous is not None:
|
||||
value = f"{previous}&{value}"
|
||||
setattr(namespace, self.dest, value)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,39 @@ class TestCurlToRequestKwargs:
|
|||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_post_data_multiple(self):
|
||||
# curl merges repeated -d/--data/--data-raw options into a single body
|
||||
# joined with "&"; scrapy must do the same, not keep only the last one.
|
||||
curl_command = "curl 'https://www.example.org/' -d 'a=1' -d 'b=2' -d 'c=3'"
|
||||
expected_result = {
|
||||
"method": "POST",
|
||||
"url": "https://www.example.org/",
|
||||
"body": "a=1&b=2&c=3",
|
||||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_post_data_multiple_mixed_flag_names(self):
|
||||
curl_command = (
|
||||
"curl 'https://www.example.org/' --data 'a=1' --data-raw 'b=2' -d 'c=3'"
|
||||
)
|
||||
expected_result = {
|
||||
"method": "POST",
|
||||
"url": "https://www.example.org/",
|
||||
"body": "a=1&b=2&c=3",
|
||||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_post_data_multiple_with_string_prefix(self):
|
||||
# The leading "$" left by bash $'...' quoting is stripped per option,
|
||||
# before the values are merged.
|
||||
curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"
|
||||
expected_result = {
|
||||
"method": "POST",
|
||||
"url": "https://www.example.org/",
|
||||
"body": "a=1&b=2",
|
||||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_explicit_get_with_data(self):
|
||||
curl_command = "curl httpbin.org/anything -X GET --data asdf"
|
||||
expected_result = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue