upgrade parsel and use its function to instantiate root for finding form

This commit is contained in:
Elias Dorneles 2015-08-07 17:33:59 -03:00
parent 2fe6d128f5
commit 26ebccd37a
3 changed files with 5 additions and 10 deletions

View File

@ -7,4 +7,4 @@ queuelib
six>=1.5.2
PyDispatcher>=2.0.5
service_identity
parsel>=0.9.2
parsel>=0.9.3

View File

@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from six.moves.urllib.parse import urljoin, urlencode
import lxml.html
from lxml import etree
from parsel.selector import create_root_node
import six
from scrapy.http.request import Request
from scrapy.utils.python import to_bytes, is_listlike
@ -55,15 +55,10 @@ def _urlencode(seq, enc):
return urlencode(values, doseq=1)
def _create_parser_from_response(response, parser_cls):
body = response.body_as_unicode().strip().encode('utf8') or b'<html/>'
parser = parser_cls(recover=True, encoding='utf8')
return etree.fromstring(body, parser=parser, base_url=response.url)
def _get_form(response, formname, formid, formnumber, formxpath):
"""Find the form element """
root = _create_parser_from_response(response, lxml.html.HTMLParser)
text = response.body_as_unicode()
root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url)
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)

View File

@ -44,7 +44,7 @@ setup(
'pyOpenSSL',
'cssselect>=0.9',
'six>=1.5.2',
'parsel>=0.9.2',
'parsel>=0.9.3',
'PyDispatcher>=2.0.5',
'service_identity',
],