diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2862dc096..905d8412f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +import six from six.moves.urllib.parse import urljoin, urlencode + import lxml.html from parsel.selector import create_root_node -import six +from w3lib.html import strip_html5_whitespace + from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url @@ -51,7 +54,10 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: - return urljoin(form.base_url, form.action) + action = form.get('action') + if action is None: + return form.base_url + return urljoin(form.base_url, strip_html5_whitespace(action)) return urljoin(form.base_url, url) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d7216e1d2..7eadb874f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -556,7 +556,6 @@ class FormRequestTest(RequestTest): fs = _qs(req, to_unicode=True, encoding='latin1') self.assertTrue(fs[u'price in \u00a5']) - def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( """
@@ -989,7 +988,7 @@ class FormRequestTest(RequestTest): """ - + @@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response) self.assertEqual(req.url, 'http://b.com/test_form') + def test_spaces_in_action(self): + resp = _buildresponse('
') + req = self.request_class.from_response(resp) + self.assertEqual(req.url, 'http://example.com/path') + def test_from_response_css(self): response = _buildresponse( """
@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formcss="input[name='abc']") + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com') kwargs.setdefault('encoding', 'utf-8') return HtmlResponse(**kwargs) + def _qs(req, encoding='utf-8', to_unicode=False): if req.method == 'POST': qs = req.body