diff --git a/requirements.txt b/requirements.txt index f8a0f7ad3..afefb9d33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=0.9.2 +parsel>=0.9.3 diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ad3f0571e..0b1d3b926 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from six.moves.urllib.parse import urljoin, urlencode import lxml.html -from lxml import etree +from parsel.selector import create_root_node import six from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike @@ -55,15 +55,10 @@ def _urlencode(seq, enc): return urlencode(values, doseq=1) -def _create_parser_from_response(response, parser_cls): - body = response.body_as_unicode().strip().encode('utf8') or b'' - parser = parser_cls(recover=True, encoding='utf8') - return etree.fromstring(body, parser=parser, base_url=response.url) - - def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ - root = _create_parser_from_response(response, lxml.html.HTMLParser) + text = response.body_as_unicode() + root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) diff --git a/setup.py b/setup.py index 2d2d8660f..4eb8d2318 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.2', + 'parsel>=0.9.3', 'PyDispatcher>=2.0.5', 'service_identity', ],