diff --git a/requirements.txt b/requirements.txt index f8a0f7ad3..afefb9d33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=0.9.2 +parsel>=0.9.3 diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ad3f0571e..0b1d3b926 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from six.moves.urllib.parse import urljoin, urlencode import lxml.html -from lxml import etree +from parsel.selector import create_root_node import six from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike @@ -55,15 +55,10 @@ def _urlencode(seq, enc): return urlencode(values, doseq=1) -def _create_parser_from_response(response, parser_cls): - body = response.body_as_unicode().strip().encode('utf8') or b'
' - parser = parser_cls(recover=True, encoding='utf8') - return etree.fromstring(body, parser=parser, base_url=response.url) - - def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ - root = _create_parser_from_response(response, lxml.html.HTMLParser) + text = response.body_as_unicode() + root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url) forms = root.xpath('//form') if not forms: raise ValueError("No