From c34dbe955d1a72ddef97afa1f0fb92becd9f4ca3 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 29 Oct 2015 14:18:59 +0800 Subject: [PATCH 1/2] fixed: Issue #1562 (Incorrectly picked URL in `scrapy.http.FormRequest.from_response` when there is a `` tag) --- scrapy/http/request/form.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a12a2fd07..4a9bd732e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -11,6 +11,7 @@ from parsel.selector import create_root_node import six from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike +from scrapy.utils.response import get_base_url class FormRequest(Request): @@ -44,7 +45,7 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: - return form.action or form.base_url + return urljoin(form.base_url, form.action) return urljoin(form.base_url, url) @@ -58,7 +59,7 @@ def _urlencode(seq, enc): def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ text = response.body_as_unicode() - root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url) + root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) From e19bf4aecc9027fd4023282a371c13f77fadd510 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Mon, 2 Nov 2015 22:52:41 +0800 Subject: [PATCH 2/2] added: Test case for the fix --- tests/test_http_request.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ff0941961..60fd855dd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -801,6 +801,25 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'test2'], [b'val2']) self.assertEqual(fs[b'button1'], [b'']) + def test_html_base_form_action(self): + response = _buildresponse( + """ + + + + + + + + + + """, + url='http://a.com/' + ) + req = self.request_class.from_response(response) + self.assertEqual(req.url, 'http://b.com/test_form') + + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com')