mirror of https://github.com/scrapy/scrapy.git
Merge dangra/lxml-formrequest
This commit is contained in:
Daniel Graña
commit
8b45a00f36
|
|
@ -32,10 +32,8 @@ class FormRequest(Request):
|
|||
@classmethod
|
||||
def from_response(cls, response, formname=None, formnumber=0, formdata=None,
|
||||
clickdata=None, dont_click=False, **kwargs):
|
||||
from scrapy.selector.lxmldocument import LxmlDocument
|
||||
kwargs.setdefault('encoding', response.encoding)
|
||||
root = LxmlDocument(response, lxml.html.HTMLParser)
|
||||
form = _get_form(root, formname, formnumber, response)
|
||||
form = _get_form(response, formname, formnumber)
|
||||
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
|
||||
url = form.action or form.base_url
|
||||
return cls(url, method=form.method, formdata=formdata, **kwargs)
|
||||
|
|
@ -47,11 +45,10 @@ def _urlencode(seq, enc):
|
|||
for v in (vs if hasattr(vs, '__iter__') else [vs])]
|
||||
return urllib.urlencode(values, doseq=1)
|
||||
|
||||
def _get_form(root, formname, formnumber, response):
|
||||
"""
|
||||
Uses all the passed arguments to get the required form
|
||||
element
|
||||
"""
|
||||
def _get_form(response, formname, formnumber):
|
||||
"""Find the form element """
|
||||
from scrapy.selector.lxmldocument import LxmlDocument
|
||||
root = LxmlDocument(response, lxml.html.HTMLParser)
|
||||
if not root.forms:
|
||||
raise ValueError("No <form> element found in %s" % response)
|
||||
|
||||
|
|
@ -77,15 +74,34 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
|
|||
except (ValueError, TypeError):
|
||||
raise ValueError('formdata should be a dict or iterable of tuples')
|
||||
|
||||
inputs = [(n, u'' if v is None else v) for n, v in form.fields.items() if n not in formdata]
|
||||
inputs = form.xpath('descendant::textarea'
|
||||
'|descendant::select'
|
||||
'|descendant::input[@type!="submit" '
|
||||
'and ((@type!="checkbox" and @type!="radio") or @checked)]')
|
||||
values = [(k, u'' if v is None else v) \
|
||||
for k, v in (_value(e) for e in inputs) \
|
||||
if k and k not in formdata]
|
||||
|
||||
if not dont_click:
|
||||
clickable = _get_clickable(clickdata, form)
|
||||
if clickable and clickable[0] not in formdata and not clickable[0] is None:
|
||||
inputs.append(clickable)
|
||||
values.append(clickable)
|
||||
|
||||
inputs.extend(formdata.iteritems())
|
||||
return inputs
|
||||
values.extend(formdata.iteritems())
|
||||
return values
|
||||
|
||||
def _value(ele):
|
||||
n = ele.name
|
||||
v = ele.value
|
||||
# Match browser behaviour on simple select tag without options selected
|
||||
# Or for select tags wihout options
|
||||
if v is None and ele.tag == 'select' and not ele.multiple:
|
||||
o = ele.value_options
|
||||
if o:
|
||||
return n, o[0]
|
||||
else:
|
||||
return None, None
|
||||
return n, v
|
||||
|
||||
def _get_clickable(clickdata, form):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -206,323 +206,388 @@ class FormRequestTest(RequestTest):
|
|||
self.assertEqual(r3.body, 'colours=red&colours=blue&colours=green&price=%C2%A3+100')
|
||||
|
||||
def test_from_response_post(self):
|
||||
respbody = """
|
||||
<form action="post.php" method="POST">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, callback=lambda x: x)
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(r1.url, "http://www.example.com/this/post.php")
|
||||
self.assertEqual(set([f.value for f in fs["test"]]), set(["val1", "val2"]))
|
||||
self.assertEqual(set([f.value for f in fs["one"]]), set(["two", "three"]))
|
||||
self.assertEqual(fs['test2'].value, 'xxx')
|
||||
self.assertEqual(fs['six'].value, 'seven')
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>""",
|
||||
url="http://www.example.com/this/list.html")
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
self.assertEqual(req.method, 'POST')
|
||||
self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.url, "http://www.example.com/this/post.php")
|
||||
fs = _qs(req)
|
||||
self.assertEqual(set(fs["test"]), set(["val1", "val2"]))
|
||||
self.assertEqual(set(fs["one"]), set(["two", "three"]))
|
||||
self.assertEqual(fs['test2'], ['xxx'])
|
||||
self.assertEqual(fs['six'], ['seven'])
|
||||
|
||||
def test_from_response_extra_headers(self):
|
||||
respbody = """
|
||||
<form action="post.php" method="POST">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>
|
||||
"""
|
||||
headers = {"Accept-Encoding": "gzip,deflate"}
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, headers=headers, callback=lambda x: x)
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
self.assertEqual(r1.headers['Accept-Encoding'], 'gzip,deflate')
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'},
|
||||
headers={"Accept-Encoding": "gzip,deflate"})
|
||||
self.assertEqual(req.method, 'POST')
|
||||
self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.headers['Accept-Encoding'], 'gzip,deflate')
|
||||
|
||||
def test_from_response_get(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>""",
|
||||
url="http://www.example.com/this/list.html")
|
||||
r1 = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
self.assertEqual(r1.method, 'GET')
|
||||
self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
|
||||
self.assertEqual(urlparse(r1.url).path, "/this/get.php")
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertEqual(set(urlargs['test']), set(['val1', 'val2']))
|
||||
self.assertEqual(set(urlargs['one']), set(['two', 'three']))
|
||||
self.assertEqual(urlargs['test2'], ['xxx'])
|
||||
self.assertEqual(urlargs['six'], ['seven'])
|
||||
fs = _qs(r1)
|
||||
self.assertEqual(set(fs['test']), set(['val1', 'val2']))
|
||||
self.assertEqual(set(fs['one']), set(['two', 'three']))
|
||||
self.assertEqual(fs['test2'], ['xxx'])
|
||||
self.assertEqual(fs['six'], ['seven'])
|
||||
|
||||
def test_from_response_override_params(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'two': '2'})
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(fs['one'].value, '1')
|
||||
self.assertEqual(fs['two'].value, '2')
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, formdata={'two': '2'})
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['one'], ['1'])
|
||||
self.assertEqual(fs['two'], ['2'])
|
||||
|
||||
def test_from_response_submit_first_clickable(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'two': '2'})
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertEqual(urlargs['clickable1'], ['clicked1'])
|
||||
self.assertFalse('clickable2' in urlargs, urlargs)
|
||||
self.assertEqual(urlargs['one'], ['1'])
|
||||
self.assertEqual(urlargs['two'], ['2'])
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, formdata={'two': '2'})
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['clickable1'], ['clicked1'])
|
||||
self.assertFalse('clickable2' in fs, fs)
|
||||
self.assertEqual(fs['one'], ['1'])
|
||||
self.assertEqual(fs['two'], ['2'])
|
||||
|
||||
def test_from_response_submit_not_first_clickable(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickable2'})
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertEqual(urlargs['clickable2'], ['clicked2'])
|
||||
self.assertFalse('clickable1' in urlargs, urlargs)
|
||||
self.assertEqual(urlargs['one'], ['1'])
|
||||
self.assertEqual(urlargs['two'], ['2'])
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, formdata={'two': '2'}, \
|
||||
clickdata={'name': 'clickable2'})
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['clickable2'], ['clicked2'])
|
||||
self.assertFalse('clickable1' in fs, fs)
|
||||
self.assertEqual(fs['one'], ['1'])
|
||||
self.assertEqual(fs['two'], ['2'])
|
||||
|
||||
def test_from_response_multiple_clickdata(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable" value="clicked1">
|
||||
<input type="submit" name="clickable" value="clicked2">
|
||||
<input type="hidden" name="one" value="clicked1">
|
||||
<input type="hidden" name="two" value="clicked2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, \
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable" value="clicked1">
|
||||
<input type="submit" name="clickable" value="clicked2">
|
||||
<input type="hidden" name="one" value="clicked1">
|
||||
<input type="hidden" name="two" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, \
|
||||
clickdata={'name': 'clickable', 'value': 'clicked2'})
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertEqual(urlargs['clickable'], ['clicked2'])
|
||||
self.assertEqual(urlargs['one'], ['clicked1'])
|
||||
self.assertEqual(urlargs['two'], ['clicked2'])
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['clickable'], ['clicked2'])
|
||||
self.assertEqual(fs['one'], ['clicked1'])
|
||||
self.assertEqual(fs['two'], ['clicked2'])
|
||||
|
||||
def test_from_response_unicode_clickdata(self):
|
||||
body = u"""
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
|
||||
<input type="submit" name="price in \u20ac" value="\u20ac 2000">
|
||||
<input type="hidden" name="poundsign" value="\u00a3">
|
||||
<input type="hidden" name="eurosign" value="\u20ac">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com", body=body, \
|
||||
encoding='utf-8')
|
||||
r1 = self.request_class.from_response(response, \
|
||||
response = _buildresponse(
|
||||
u"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
|
||||
<input type="submit" name="price in \u20ac" value="\u20ac 2000">
|
||||
<input type="hidden" name="poundsign" value="\u00a3">
|
||||
<input type="hidden" name="eurosign" value="\u20ac">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, \
|
||||
clickdata={'name': u'price in \u00a3'})
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertTrue(urlargs[u'price in \u00a3'.encode('utf-8')])
|
||||
|
||||
def test_from_response_with_select(self):
|
||||
body = u"""
|
||||
<form name="form1">
|
||||
<select name="inputname"><option selected="selected" value="inputvalue">text</option></select>
|
||||
<input type="submit" name="clickable" value="clicked">
|
||||
</form>
|
||||
"""
|
||||
res = HtmlResponse("http://example.com", body=body, encoding='utf-8')
|
||||
req = self.request_class.from_response(res)
|
||||
urlargs = cgi.parse_qs(urlparse(req.url).query)
|
||||
self.assertEqual(urlargs['inputname'], ['inputvalue'])
|
||||
fs = _qs(req)
|
||||
self.assertTrue(fs[u'price in \u00a3'.encode('utf-8')])
|
||||
|
||||
def test_from_response_multiple_forms_clickdata(self):
|
||||
body = u"""
|
||||
<form name="form1">
|
||||
<input type="submit" name="clickable" value="clicked1">
|
||||
<input type="hidden" name="field1" value="value1">
|
||||
</form>
|
||||
<form name="form2">
|
||||
<input type="submit" name="clickable" value="clicked2">
|
||||
<input type="hidden" name="field2" value="value2">
|
||||
</form>
|
||||
"""
|
||||
res = HtmlResponse("http://example.com", body=body, encoding='utf-8')
|
||||
req = self.request_class.from_response(res, formname='form2', \
|
||||
response = _buildresponse(
|
||||
"""<form name="form1">
|
||||
<input type="submit" name="clickable" value="clicked1">
|
||||
<input type="hidden" name="field1" value="value1">
|
||||
</form>
|
||||
<form name="form2">
|
||||
<input type="submit" name="clickable" value="clicked2">
|
||||
<input type="hidden" name="field2" value="value2">
|
||||
</form>
|
||||
""")
|
||||
req = self.request_class.from_response(response, formname='form2', \
|
||||
clickdata={'name': 'clickable'})
|
||||
urlargs = cgi.parse_qs(urlparse(req.url).query)
|
||||
self.assertEqual(urlargs['clickable'], ['clicked2'])
|
||||
self.assertEqual(urlargs['field2'], ['value2'])
|
||||
self.assertFalse('field1' in urlargs, urlargs)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['clickable'], ['clicked2'])
|
||||
self.assertEqual(fs['field2'], ['value2'])
|
||||
self.assertFalse('field1' in fs, fs)
|
||||
|
||||
def test_from_response_override_clickable(self):
|
||||
body = u'<form><input type="submit" name="clickme" value="one"></form>'
|
||||
res = HtmlResponse("http://example.com", body=body, encoding='utf-8')
|
||||
req = self.request_class.from_response(res, \
|
||||
formdata={'clickme': 'two'}, \
|
||||
clickdata={'name': 'clickme'})
|
||||
urlargs = cgi.parse_qs(urlparse(req.url).query)
|
||||
self.assertEqual(urlargs['clickme'], ['two'])
|
||||
response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
|
||||
req = self.request_class.from_response(response, \
|
||||
formdata={'clickme': 'two'}, clickdata={'name': 'clickme'})
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs['clickme'], ['two'])
|
||||
|
||||
def test_from_response_dont_click(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>""")
|
||||
r1 = self.request_class.from_response(response, dont_click=True)
|
||||
urlargs = cgi.parse_qs(urlparse(r1.url).query)
|
||||
self.assertFalse('clickable1' in urlargs, urlargs)
|
||||
self.assertFalse('clickable2' in urlargs, urlargs)
|
||||
fs = _qs(r1)
|
||||
self.assertFalse('clickable1' in fs, fs)
|
||||
self.assertFalse('clickable2' in fs, fs)
|
||||
|
||||
def test_from_response_ambiguous_clickdata(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
|
||||
self.assertRaises(ValueError,
|
||||
self.request_class.from_response,
|
||||
response,
|
||||
clickdata={'type': 'submit'})
|
||||
response = _buildresponse(
|
||||
"""
|
||||
<form action="get.php" method="GET">
|
||||
<input type="submit" name="clickable1" value="clicked1">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>""")
|
||||
self.assertRaises(ValueError, self.request_class.from_response,
|
||||
response, clickdata={'type': 'submit'})
|
||||
|
||||
def test_from_response_non_matching_clickdata(self):
|
||||
body = """
|
||||
<form>
|
||||
<input type="submit" name="clickable" value="clicked">
|
||||
</form>
|
||||
"""
|
||||
res = HtmlResponse("http://example.com", body=body)
|
||||
self.assertRaises(ValueError,
|
||||
self.request_class.from_response, res,
|
||||
clickdata={'nonexistent': 'notme'})
|
||||
response = _buildresponse(
|
||||
"""<form>
|
||||
<input type="submit" name="clickable" value="clicked">
|
||||
</form>""")
|
||||
self.assertRaises(ValueError, self.request_class.from_response,
|
||||
response, clickdata={'nonexistent': 'notme'})
|
||||
|
||||
def test_from_response_errors_noform(self):
|
||||
respbody = """<html></html>"""
|
||||
response = HtmlResponse("http://www.example.com/lala.html", body=respbody)
|
||||
response = _buildresponse("""<html></html>""")
|
||||
self.assertRaises(ValueError, self.request_class.from_response, response)
|
||||
|
||||
def test_from_response_errors_formnumber(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="GET">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/lala.html", body=respbody)
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="hidden" name="test" value="val1">
|
||||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>""")
|
||||
self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1)
|
||||
|
||||
def test_from_response_noformname(self):
|
||||
respbody = """
|
||||
<form action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/formname.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formdata={'two':'3'}, callback=lambda x: x)
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>""")
|
||||
r1 = self.request_class.from_response(response, formdata={'two':'3'})
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(fs['one'].value, '1')
|
||||
self.assertEqual(fs['two'].value, '3')
|
||||
|
||||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {'one': ['1'], 'two': ['3']})
|
||||
|
||||
def test_from_response_formname_exists(self):
|
||||
respbody = """
|
||||
<form action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="three" value="3">
|
||||
<input type="hidden" name="four" value="4">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/formname.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formname="form2", callback=lambda x: x)
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="three" value="3">
|
||||
<input type="hidden" name="four" value="4">
|
||||
</form>""")
|
||||
r1 = self.request_class.from_response(response, formname="form2")
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(fs['three'].value, "3")
|
||||
self.assertEqual(fs['four'].value, "4")
|
||||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
|
||||
|
||||
def test_from_response_formname_notexist(self):
|
||||
respbody = """
|
||||
<form name="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/formname.html", body=respbody)
|
||||
r1 = self.request_class.from_response(response, formname="form3", callback=lambda x: x)
|
||||
response = _buildresponse(
|
||||
"""<form name="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>""")
|
||||
r1 = self.request_class.from_response(response, formname="form3")
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(fs['one'].value, "1")
|
||||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {'one': ['1']})
|
||||
|
||||
def test_from_response_formname_errors_formnumber(self):
|
||||
respbody = """
|
||||
<form name="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>
|
||||
"""
|
||||
response = HtmlResponse("http://www.example.com/formname.html", body=respbody)
|
||||
self.assertRaises(IndexError, self.request_class.from_response, response, formname="form3", formnumber=2)
|
||||
response = _buildresponse(
|
||||
"""<form name="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
</form>
|
||||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>""")
|
||||
self.assertRaises(IndexError, self.request_class.from_response, \
|
||||
response, formname="form3", formnumber=2)
|
||||
|
||||
def test_from_response_missed_value(self):
|
||||
respbody = """
|
||||
<form action="get.php" method="POST">
|
||||
<input type="hidden" name="key1" value="val1">
|
||||
<input type="hidden" name="key2">
|
||||
<input type="radio" name="key3" checked>
|
||||
<input type="checkbox" name="key4" checked>
|
||||
<textarea name="key5">val5</textarea>
|
||||
<textarea name="key6"/>
|
||||
<select name="key7">
|
||||
<option name="opt1" selected></option>
|
||||
<option name="opt2">opt2</option>
|
||||
</select>
|
||||
<input type="submit" value="val8">
|
||||
</form>
|
||||
"""
|
||||
res = HtmlResponse("http://example.com", body=respbody, encoding='utf-8')
|
||||
def test_from_response_select(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<select name="i1">
|
||||
<option value="i1v1">option 1</option>
|
||||
<option value="i1v2" selected>option 2</option>
|
||||
</select>
|
||||
<select name="i2">
|
||||
<option value="i2v1">option 1</option>
|
||||
<option value="i2v2">option 2</option>
|
||||
</select>
|
||||
<select>
|
||||
<option value="i3v1">option 1</option>
|
||||
<option value="i3v2">option 2</option>
|
||||
</select>
|
||||
<select name="i4" multiple>
|
||||
<option value="i4v1">option 1</option>
|
||||
<option value="i4v2" selected>option 2</option>
|
||||
<option value="i4v3" selected>option 3</option>
|
||||
</select>
|
||||
<select name="i5" multiple>
|
||||
<option value="i5v1">option 1</option>
|
||||
<option value="i5v2">option 2</option>
|
||||
</select>
|
||||
<select name="i6"></select>
|
||||
<select name="i7"/>
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = cgi.parse_qs(req.body, True)
|
||||
self.assertEqual(fs['key1'], ['val1'])
|
||||
self.assertEqual(fs['key2'], [''])
|
||||
self.assertEqual(fs['key3'], ['on'])
|
||||
self.assertEqual(fs['key4'], ['on'])
|
||||
self.assertEqual(fs['key5'], ['val5'])
|
||||
self.assertEqual(fs['key6'], [''])
|
||||
self.assertEqual(fs['key7'], [''])
|
||||
self.assertEqual(set(fs), set(['key1', 'key2', 'key3', 'key4', 'key5', 'key6', 'key7']))
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']})
|
||||
|
||||
def test_from_response_radio(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<input type="radio" name="i1" value="i1v1">
|
||||
<input type="radio" name="i1" value="iv2" checked>
|
||||
<input type="radio" name="i2" checked>
|
||||
<input type="radio" name="i2">
|
||||
<input type="radio" name="i3" value="i3v1">
|
||||
<input type="radio" name="i3">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']})
|
||||
|
||||
def test_from_response_checkbox(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<input type="checkbox" name="i1" value="i1v1">
|
||||
<input type="checkbox" name="i1" value="iv2" checked>
|
||||
<input type="checkbox" name="i2" checked>
|
||||
<input type="checkbox" name="i2">
|
||||
<input type="checkbox" name="i3" value="i3v1">
|
||||
<input type="checkbox" name="i3">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']})
|
||||
|
||||
def test_from_response_input_text(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<input type="text" name="i1" value="i1v1">
|
||||
<input type="text" name="i2">
|
||||
<input type="text">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
|
||||
|
||||
def test_from_response_input_hidden(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<input type="hidden" name="i1" value="i1v1">
|
||||
<input type="hidden" name="i2">
|
||||
<input type="hidden">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
|
||||
|
||||
def test_from_response_input_hidden(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<input type="hidden" name="i1" value="i1v1">
|
||||
<input type="hidden" name="i2">
|
||||
<input type="hidden">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
|
||||
|
||||
def test_from_response_input_textarea(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<textarea name="i1">i1v</textarea>
|
||||
<textarea name="i2"></textarea>
|
||||
<textarea name="i3"/>
|
||||
<textarea>i4v</textarea>
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {'i1': ['i1v'], 'i2': [''], 'i3': ['']})
|
||||
|
||||
def test_from_response_descendants(self):
|
||||
res = _buildresponse(
|
||||
'''<form>
|
||||
<div>
|
||||
<fieldset>
|
||||
<input type="text" name="i1">
|
||||
<select name="i2">
|
||||
<option value="v1" selected>
|
||||
</select>
|
||||
</fieldset>
|
||||
<input type="radio" name="i3" value="i3v2" checked>
|
||||
<input type="checkbox" name="i4" value="i4v2" checked>
|
||||
<textarea name="i5"></textarea>
|
||||
<input type="hidden" name="h1" value="h1v">
|
||||
</div>
|
||||
<input type="hidden" name="h2" value="h2v">
|
||||
</form>''')
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(set(fs), set(['h2', 'i2', 'i1', 'i3', 'h1', 'i5', 'i4']))
|
||||
|
||||
|
||||
def _buildresponse(body, **kwargs):
|
||||
kwargs.setdefault('body', body)
|
||||
kwargs.setdefault('url', 'http://example.com')
|
||||
kwargs.setdefault('encoding', 'utf-8')
|
||||
return HtmlResponse(**kwargs)
|
||||
|
||||
def _qs(req):
|
||||
if req.method == 'POST':
|
||||
qs = req.body
|
||||
else:
|
||||
qs = req.url.partition('?')[2]
|
||||
return cgi.parse_qs(qs, True)
|
||||
|
||||
|
||||
class XmlRpcRequestTest(RequestTest):
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue