diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index aa755b812..53470825f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -31,9 +31,9 @@ class FormRequest(Request): @classmethod def from_response(cls, response, formname=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, **kwargs): + clickdata=None, dont_click=False, formxpath=None, **kwargs): kwargs.setdefault('encoding', response.encoding) - form = _get_form(response, formname, formnumber) + form = _get_form(response, formname, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = form.action or form.base_url return cls(url, method=form.method, formdata=formdata, **kwargs) @@ -45,7 +45,7 @@ def _urlencode(seq, enc): for v in (vs if hasattr(vs, '__iter__') else [vs])] return urllib.urlencode(values, doseq=1) -def _get_form(response, formname, formnumber): +def _get_form(response, formname, formnumber, formxpath): """Find the form element """ from scrapy.selector.lxmldocument import LxmlDocument root = LxmlDocument(response, lxml.html.HTMLParser) @@ -56,6 +56,19 @@ def _get_form(response, formname, formnumber): f = root.xpath('//form[@name="%s"]' % formname) if f: return f[0] + + # Get form element from xpath, if not found, go up + if formxpath is not None: + nodes = root.xpath(formxpath) + if nodes: + el = nodes[0] + while True: + if el.tag == 'form': + return el + el = el.getparent() + if el is None: + break + raise ValueError('No
+ """) + r1 = self.request_class.from_response(response, formxpath="//form[@action='post.php']") + fs = _qs(r1) + self.assertEqual(fs['one'], ['1']) + + r1 = self.request_class.from_response(response, formxpath="//form/input[@name='four']") + fs = _qs(r1) + self.assertEqual(fs['three'], ['3']) + + self.assertRaises(ValueError, self.request_class.from_response, + response, formxpath="//form/input[@name='abc']") def _buildresponse(body, **kwargs): kwargs.setdefault('body', body)