RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1137 from DharmeshPandav/patch-1 

Update form.py to improve existing capability
This commit is contained in:
Daniel Graña 2015-07-31 11:00:57 -03:00
parent 936266a61a 954c8fcecb
commit 786f62664b
2 changed files with 62 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapy/http/request/form.py
View File

@ -31,10 +31,10 @@ class FormRequest(Request):
self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
@classmethod
def from_response(cls, response, formname=None, formnumber=0, formdata=None,
def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
clickdata=None, dont_click=False, formxpath=None, **kwargs):
kwargs.setdefault('encoding', response.encoding)
form = _get_form(response, formname, formnumber, formxpath)
form = _get_form(response, formname, formid, formnumber, formxpath)
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
url = _get_form_url(form, kwargs.pop('url', None))
method = kwargs.pop('method', form.method)
@ -54,7 +54,7 @@ def _urlencode(seq, enc):
return urlencode(values, doseq=1)
def _get_form(response, formname, formnumber, formxpath):
def _get_form(response, formname, formid, formnumber, formxpath):
"""Find the form element """
from scrapy.selector.lxmldocument import LxmlDocument
root = LxmlDocument(response, lxml.html.HTMLParser)
@ -67,6 +67,11 @@ def _get_form(response, formname, formnumber, formxpath):
if f:
return f[0]
if formid is not None:
f = root.xpath('//form[@id="%s"]' % formid)
if f:
return f[0]
# Get form element from xpath, if not found, go up
if formxpath is not None:
nodes = root.xpath(formxpath)

54
tests/test_http_request.py
View File

@ -538,6 +538,60 @@ class FormRequestTest(RequestTest):
self.assertRaises(IndexError, self.request_class.from_response, \
response, formname="form3", formnumber=2)
def test_from_response_formid_exists(self):
response = _buildresponse(
"""<form action="post.php" method="POST">
<input type="hidden" name="one" value="1">
<input type="hidden" name="two" value="2">
</form>
<form id="form2" action="post.php" method="POST">
<input type="hidden" name="three" value="3">
<input type="hidden" name="four" value="4">
</form>""")
r1 = self.request_class.from_response(response, formid="form2")
self.assertEqual(r1.method, 'POST')
fs = _qs(r1)
self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
def test_from_response_formname_notexists_fallback_formid(self):
response = _buildresponse(
"""<form action="post.php" method="POST">
<input type="hidden" name="one" value="1">
<input type="hidden" name="two" value="2">
</form>
<form id="form2" name="form2" action="post.php" method="POST">
<input type="hidden" name="three" value="3">
<input type="hidden" name="four" value="4">
</form>""")
r1 = self.request_class.from_response(response, formname="form3", formid="form2")
self.assertEqual(r1.method, 'POST')
fs = _qs(r1)
self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
def test_from_response_formid_notexist(self):
response = _buildresponse(
"""<form id="form1" action="post.php" method="POST">
<input type="hidden" name="one" value="1">
</form>
<form id="form2" action="post.php" method="POST">
<input type="hidden" name="two" value="2">
</form>""")
r1 = self.request_class.from_response(response, formid="form3")
self.assertEqual(r1.method, 'POST')
fs = _qs(r1)
self.assertEqual(fs, {'one': ['1']})
def test_from_response_formid_errors_formnumber(self):
response = _buildresponse(
"""<form id="form1" action="post.php" method="POST">
<input type="hidden" name="one" value="1">
</form>
<form id="form2" name="form2" action="post.php" method="POST">
<input type="hidden" name="two" value="2">
</form>""")
self.assertRaises(IndexError, self.request_class.from_response, \
response, formid="form3", formnumber=2)
def test_from_response_select(self):
res = _buildresponse(
'''<form>