From 558a1a5033a2c01074af10a3a5fc374f064c0169 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 20 Apr 2009 02:29:19 +0000 Subject: [PATCH] FormRequest.from_response: fixed bug with setting method, override fields with those included in formdata, for those which already existed in the response
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401070 --- scrapy/trunk/scrapy/http/request/form.py | 7 ++++++- scrapy/trunk/scrapy/tests/test_http_request.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/http/request/form.py b/scrapy/trunk/scrapy/http/request/form.py index 56322546c..2864ac030 100644 --- a/scrapy/trunk/scrapy/http/request/form.py +++ b/scrapy/trunk/scrapy/http/request/form.py @@ -45,9 +45,14 @@ class FormRequest(Request): except IndexError: raise IndexError("Form number %d not found in %s" % (formnumber, response)) if formdata: + # remove all existing fields with the same name before, so that + # formdata fields properly can properly override existing ones, + # which is the desired behaviour + form.controls = [c for c in form.controls if c.name not in formdata.keys()] for k, v in formdata.iteritems(): for v2 in v if hasattr(v, '__iter__') else [v]: form.new_control('text', k, {'value': v2}) + url, body, headers = form.click_request_data() - request = cls(url, body=body, headers=headers, **kwargs) + request = cls(url, method=form.method, body=body, headers=headers, **kwargs) return request diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index 5c99baa7a..4e3f729bc 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -209,6 +209,8 @@ class FormRequestTest(unittest.TestCase): """ response = Response("http://www.example.com/this/list.html", body=respbody) r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, callback=lambda x: x) + self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded') fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) self.assertEqual(r1.url, "http://www.example.com/this/post.php") self.assertEqual(set([f.value for f in fs["test"]]), set(["val1", "val2"])) @@ -226,6 +228,7 @@ class FormRequestTest(unittest.TestCase): """ response = Response("http://www.example.com/this/list.html", body=respbody) r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + self.assertEqual(r1.method, 'GET') self.assertEqual(r1.url.hostname, "www.example.com") self.assertEqual(r1.url.path, "/this/get.php") urlargs = cgi.parse_qs(r1.url.query) @@ -234,6 +237,19 @@ class FormRequestTest(unittest.TestCase): self.assertEqual(urlargs['test2'], ['xxx']) self.assertEqual(urlargs['six'], ['seven']) + def test_from_response_override_params(self): + respbody = """ + + + +
+ """ + response = Response("http://www.example.com/this/list.html", body=respbody) + r1 = FormRequest.from_response(response, formdata={'two': '2'}) + fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) + self.assertEqual(fs['one'].value, '1') + self.assertEqual(fs['two'].value, '2') + def test_from_response_errors_noform(self): respbody = """""" response = Response("http://www.example.com/lala.html", body=respbody)