Merge pull request #111 from LucianU/lxml-formrequest

Lxml formrequest

scrapy/http/request/form.py

@@ -6,70 +6,123 @@ See documentation in docs/topics/request-response.rst
 """
 
 import urllib
-from cStringIO import StringIO
-
-from scrapy.xlib.ClientForm import ParseFile
-
+import lxml.html
 from scrapy.http.request import Request
 from scrapy.utils.python import unicode_to_str
 
-def _unicode_to_str(string, encoding):
-    if hasattr(string, '__iter__'):
-        return [unicode_to_str(k, encoding) for k in string]
-    else:
-        return unicode_to_str(string, encoding)
-
 
 class FormRequest(Request):
 
     def __init__(self, *args, **kwargs):
         formdata = kwargs.pop('formdata', None)
+        if formdata and kwargs.get('method') is None:
+            kwargs['method'] = 'POST'
+
         super(FormRequest, self).__init__(*args, **kwargs)
 
         if formdata:
             items = formdata.iteritems() if isinstance(formdata, dict) else formdata
-            query = [(unicode_to_str(k, self.encoding), _unicode_to_str(v, self.encoding))
-                    for k, v in items]
-            self.method = 'POST'
-            self._set_body(urllib.urlencode(query, doseq=1))
-            self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+            querystr = _urlencode(items, self.encoding)
+            if self.method == 'POST':
+                self.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
+                self._set_body(querystr)
+            else:
+                self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
 
     @classmethod
-    def from_response(cls, response, formname=None, formnumber=0, formdata=None, 
+    def from_response(cls, response, formname=None, formnumber=0, formdata=None,
                       clickdata=None, dont_click=False, **kwargs):
-        encoding = getattr(response, 'encoding', 'utf-8')
-        forms = ParseFile(StringIO(response.body), response.url,
-                          encoding=encoding, backwards_compat=False)
-        if not forms:
-            raise ValueError("No <form> element found in %s" % response)
-        
-        form = None
+        from scrapy.selector.lxmldocument import LxmlDocument
+        kwargs.setdefault('encoding', response.encoding)
+        root = LxmlDocument(response, lxml.html.HTMLParser)
+        form = _get_form(root, formname, formnumber, response)
+        formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
+        url = form.action or form.base_url
+        return cls(url, method=form.method, formdata=formdata, **kwargs)
 
-        if formname:
-            for f in forms:
-                if f.name == formname:
-                    form = f
-                    break
 
-        if not form:
-            try:
-                form = forms[formnumber]
-            except IndexError:
-                raise IndexError("Form number %d not found in %s" % (formnumber, response))
-        if formdata:
-            # remove all existing fields with the same name before, so that
-            # formdata fields properly can properly override existing ones,
-            # which is the desired behaviour
-            form.controls = [c for c in form.controls if c.name not in formdata]
-            for k, v in formdata.iteritems():
-                for v2 in v if hasattr(v, '__iter__') else [v]:
-                    form.new_control('text', k, {'value': v2})
+def _urlencode(seq, enc):
+    values = [(unicode_to_str(k, enc), unicode_to_str(v, enc))
+              for k, vs in seq
+              for v in (vs if hasattr(vs, '__iter__') else [vs])]
+    return urllib.urlencode(values, doseq=1)
 
-        if dont_click:
-            url, body, headers = form._switch_click('request_data')
+def _get_form(root, formname, formnumber, response):
+    """
+    Uses all the passed arguments to get the required form
+    element
+    """
+    if not root.forms:
+        raise ValueError("No <form> element found in %s" % response)
+
+    if formname is not None:
+        f = root.xpath('//form[@name="%s"]' % formname)
+        if f:
+            return f[0]
+
+    # If we get here, it means that either formname was None
+    # or invalid
+    if formnumber is not None:
+        try:
+            form = root.forms[formnumber]
+        except IndexError:
+            raise IndexError("Form number %d not found in %s" %
+                                (formnumber, response))
         else:
-            url, body, headers = form.click_request_data(**(clickdata or {}))
+            return form
 
-        kwargs.setdefault('headers', {}).update(headers)
+def _get_inputs(form, formdata, dont_click, clickdata, response):
+    try:
+        formdata = dict(formdata or ())
+    except (ValueError, TypeError):
+        raise ValueError('formdata should be a dict or iterable of tuples')
 
-        return cls(url, method=form.method, body=body, **kwargs)
+    inputs = [(n, v) for n, v in form.form_values() if n not in formdata]
+
+    if not dont_click:
+        clickable = _get_clickable(clickdata, form)
+        if clickable and clickable[0] not in formdata:
+            inputs.append(clickable)
+
+    inputs.extend(formdata.iteritems())
+    return inputs
+
+def _get_clickable(clickdata, form):
+    """
+    Returns the clickable element specified in clickdata,
+    if the latter is given. If not, it returns the first
+    clickable element found
+    """
+    clickables = [el for el in form.inputs if el.type == 'submit']
+    if not clickables:
+        return
+
+    # If we don't have clickdata, we just use the first clickable element
+    if clickdata is None:
+        el = clickables[0]
+        return (el.name, el.value)
+
+    # If clickdata is given, we compare it to the clickable elements to find a
+    # match. We first look to see if the number is specified in clickdata,
+    # because that uniquely identifies the element
+    nr = clickdata.get('nr', None)
+    if nr is not None:
+        try:
+            el = list(form.inputs)[nr]
+        except IndexError:
+            pass
+        else:
+            return (el.name, el.value)
+
+    # We didn't find it, so now we build an XPath expression out of the other
+    # arguments, because they can be used as such
+    xpath = u'.//*' + \
+            u''.join(u'[@%s="%s"]' % c for c in clickdata.iteritems())
+    el = form.xpath(xpath)
+    if len(el) == 1:
+        return (el[0].name, el[0].value)
+    elif len(el) > 1:
+        raise ValueError("Multiple elements found (%r) matching the criteria "
+                         "in clickdata: %r" % (el, clickdata))
+    else:
+        raise ValueError('No clickable element matching clickdata: %r' % (clickdata,))

scrapy/selector/lxmldocument.py

@@ -18,7 +18,7 @@ def _factory(response, parser_cls):
 class LxmlDocument(object_ref):
 
     cache = weakref.WeakKeyDictionary()
-    __slots__ = ['xmlDoc', 'xpathContext', '__weakref__']
+    __slots__ = ['__weakref__']
 
     def __new__(cls, response, parser=etree.HTMLParser):
         cache = cls.cache.setdefault(response, {})

scrapy/tests/test_clientform.py

@@ -1,8 +0,0 @@
-import unittest
-
-from scrapy.xlib import ClientForm
-
-class ClientFormPatchTests(unittest.TestCase):
-
-    def test_patched_unescape_charref(self):
-        self.assertEqual(ClientForm.unescape_charref('c', 'utf-8'), 'c')

scrapy/tests/test_http_request.py

@@ -271,54 +271,142 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs['one'].value, '1')
         self.assertEqual(fs['two'].value, '2')
 
-    def test_from_response_submit_first_clickeable(self):
+    def test_from_response_submit_first_clickable(self):
         respbody = """
 <form action="get.php" method="GET">
-<input type="submit" name="clickeable1" value="clicked1">
+<input type="submit" name="clickable1" value="clicked1">
 <input type="hidden" name="one" value="1">
 <input type="hidden" name="two" value="3">
-<input type="submit" name="clickeable2" value="clicked2">
+<input type="submit" name="clickable2" value="clicked2">
 </form>
         """
         response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
         r1 = self.request_class.from_response(response, formdata={'two': '2'})
         urlargs = cgi.parse_qs(urlparse(r1.url).query)
-        self.assertEqual(urlargs['clickeable1'], ['clicked1'])
-        self.assertFalse('clickeable2' in urlargs, urlargs)
+        self.assertEqual(urlargs['clickable1'], ['clicked1'])
+        self.assertFalse('clickable2' in urlargs, urlargs)
         self.assertEqual(urlargs['one'], ['1'])
         self.assertEqual(urlargs['two'], ['2'])
 
-    def test_from_response_submit_not_first_clickeable(self):
+    def test_from_response_submit_not_first_clickable(self):
         respbody = """
 <form action="get.php" method="GET">
-<input type="submit" name="clickeable1" value="clicked1">
+<input type="submit" name="clickable1" value="clicked1">
 <input type="hidden" name="one" value="1">
 <input type="hidden" name="two" value="3">
-<input type="submit" name="clickeable2" value="clicked2">
+<input type="submit" name="clickable2" value="clicked2">
 </form>
         """
         response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
-        r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickeable2'})
+        r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickable2'})
         urlargs = cgi.parse_qs(urlparse(r1.url).query)
-        self.assertEqual(urlargs['clickeable2'], ['clicked2'])
-        self.assertFalse('clickeable1' in urlargs, urlargs)
+        self.assertEqual(urlargs['clickable2'], ['clicked2'])
+        self.assertFalse('clickable1' in urlargs, urlargs)
         self.assertEqual(urlargs['one'], ['1'])
         self.assertEqual(urlargs['two'], ['2'])
 
+    def test_from_response_multiple_clickdata(self):
+        respbody = """
+<form action="get.php" method="GET">
+<input type="submit" name="clickable" value="clicked1">
+<input type="submit" name="clickable" value="clicked2">
+<input type="hidden" name="one" value="clicked1">
+<input type="hidden" name="two" value="clicked2">
+</form>
+        """
+        response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
+        r1 = self.request_class.from_response(response, \
+                clickdata={'name': 'clickable', 'value': 'clicked2'})
+        urlargs = cgi.parse_qs(urlparse(r1.url).query)
+        self.assertEqual(urlargs['clickable'], ['clicked2'])
+        self.assertEqual(urlargs['one'], ['clicked1'])
+        self.assertEqual(urlargs['two'], ['clicked2'])
+
+    def test_from_response_unicode_clickdata(self):
+        body = u"""
+<form action="get.php" method="GET">
+<input type="submit" name="price in \u00a3" value="\u00a3 1000">
+<input type="submit" name="price in \u20ac" value="\u20ac 2000">
+<input type="hidden" name="poundsign" value="\u00a3">
+<input type="hidden" name="eurosign" value="\u20ac">
+</form>
+        """
+        response = HtmlResponse("http://www.example.com", body=body, \
+                                encoding='utf-8')
+        r1 = self.request_class.from_response(response, \
+                clickdata={'name': u'price in \u00a3'})
+        urlargs = cgi.parse_qs(urlparse(r1.url).query)
+        self.assertTrue(urlargs[u'price in \u00a3'.encode('utf-8')])
+
+    def test_from_response_multiple_forms_clickdata(self):
+        body = u"""
+        <form name="form1">
+          <input type="submit" name="clickable" value="clicked">
+          <input type="hidden" name="field1" value="value1">
+        </form>
+        <form name="form2">
+          <input type="submit" name="clickable" value="clicked">
+          <input type="hidden" name="field2" value="value2">
+        </form>
+        """
+        res = HtmlResponse("http://example.com", body=body, encoding='utf-8')
+        req = self.request_class.from_response(res, \
+                                              formname='form2', \
+                                              clickdata={'name': 'clickable'})
+        urlargs = cgi.parse_qs(urlparse(req.url).query)
+        self.assertEqual(urlargs['clickable'], ['clicked'])
+        self.assertEqual(urlargs['field2'], ['value2'])
+        self.assertFalse('field1' in urlargs, urlargs)
+
+    def test_from_response_multiple_forms_clickdata(self):
+        body = u'<form><input type="submit" name="clickme" value="one"></form>'
+        res = HtmlResponse("http://example.com", body=body, encoding='utf-8')
+        req = self.request_class.from_response(res, \
+                                               formdata={'clickme': 'two'}, \
+                                               clickdata={'name': 'clickme'})
+        urlargs = cgi.parse_qs(urlparse(req.url).query)
+        self.assertEqual(urlargs['clickme'], ['two'])
+
     def test_from_response_dont_click(self):
         respbody = """
 <form action="get.php" method="GET">
-<input type="submit" name="clickeable1" value="clicked1">
+<input type="submit" name="clickable1" value="clicked1">
 <input type="hidden" name="one" value="1">
 <input type="hidden" name="two" value="3">
-<input type="submit" name="clickeable2" value="clicked2">
+<input type="submit" name="clickable2" value="clicked2">
 </form>
         """
         response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
         r1 = self.request_class.from_response(response, dont_click=True)
         urlargs = cgi.parse_qs(urlparse(r1.url).query)
-        self.assertFalse('clickeable1' in urlargs, urlargs)
-        self.assertFalse('clickeable2' in urlargs, urlargs)
+        self.assertFalse('clickable1' in urlargs, urlargs)
+        self.assertFalse('clickable2' in urlargs, urlargs)
+
+    def test_from_response_ambiguous_clickdata(self):
+        respbody = """
+<form action="get.php" method="GET">
+<input type="submit" name="clickable1" value="clicked1">
+<input type="hidden" name="one" value="1">
+<input type="hidden" name="two" value="3">
+<input type="submit" name="clickable2" value="clicked2">
+</form>
+        """
+        response = HtmlResponse("http://www.example.com/this/list.html", body=respbody)
+        self.assertRaises(ValueError,
+                          self.request_class.from_response,
+                          response,
+                          clickdata={'type': 'submit'})
+
+    def test_from_response_non_matching_clickdata(self):
+        body = """
+        <form>
+          <input type="submit" name="clickable" value="clicked">
+        </form>
+        """
+        res = HtmlResponse("http://example.com", body=body)
+        self.assertRaises(ValueError,
+                          self.request_class.from_response, res,
+                                clickdata={'nonexistent': 'notme'})
 
     def test_from_response_errors_noform(self):
         respbody = """<html></html>"""

scrapy/xlib/ClientForm.patch

@@ -1,15 +0,0 @@
-diff --git a/scrapy/xlib/ClientForm.py b/scrapy/xlib/ClientForm.py
---- a/scrapy/xlib/ClientForm.py
-+++ b/scrapy/xlib/ClientForm.py
-@@ -242,5 +242,10 @@ def unescape_charref(data, encoding):
-     if name.startswith("x"):
-         name, base= name[1:], 16
--    uc = unichr(int(name, base))
-+    try:
-+        uc = unichr(int(name, base))
-+    except ValueError:
-+        # invalid literal for int()
-+        # or integer not in unichr()'s range
-+        uc = name
-     if encoding is None:
-         return uc