RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Find form nodes in invalid html5 documents 

lxml fails to parse invalid html5 documents
This error was reported in scrapy/loginform#3
This commit is contained in:
Daniel Graña 2013-01-24 17:47:36 -02:00
parent ff04480675
commit 8e77f27897
2 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/http/request/form.py
View File

@ -49,14 +49,15 @@ def _get_form(response, formname, formnumber, formxpath):
"""Find the form element """
from scrapy.selector.lxmldocument import LxmlDocument
root = LxmlDocument(response, lxml.html.HTMLParser)
if not root.forms:
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)
if formname is not None:
f = root.xpath('//form[@name="%s"]' % formname)
if f:
return f[0]
# Get form element from xpath, if not found, go up
if formxpath is not None:
nodes = root.xpath(formxpath)
@ -74,7 +75,7 @@ def _get_form(response, formname, formnumber, formxpath):
# or invalid
if formnumber is not None:
try:
form = root.forms[formnumber]
form = forms[formnumber]
except IndexError:
raise IndexError("Form number %d not found in %s" %
(formnumber, response))

10
scrapy/tests/test_http_request.py
View File

@ -116,7 +116,7 @@ class RequestTest(unittest.TestCase):
def test_copy(self):
"""Test Request copy"""
def somecallback():
pass
@ -400,6 +400,14 @@ class FormRequestTest(RequestTest):
response = _buildresponse("""<html></html>""")
self.assertRaises(ValueError, self.request_class.from_response, response)
def test_from_response_invalid_html5(self):
response = _buildresponse("""<!DOCTYPE html><body></html><form>"""
"""<input type="text" name="foo" value="xxx">"""
"""</form></body></html>""")
req = self.request_class.from_response(response, formdata={'bar': 'buz'})
fs = _qs(req)
self.assertEqual(fs, {'foo': ['xxx'], 'bar': ['buz']})
def test_from_response_errors_formnumber(self):
response = _buildresponse(
"""<form action="get.php" method="GET">