RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1563 from starrify/master 

[MRG+1] fixed: Issue #1562 (Incorrectly picked URL in `scrapy.http.FormReques t.from_response` when there is a `<base>` tag)
This commit is contained in:
Mikhail Korobov 2015-11-10 21:33:24 +05:00
parent 57f87b95d4 e19bf4aecc
commit 9548691fdd
2 changed files with 22 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/http/request/form.py
View File

@ -11,6 +11,7 @@ from parsel.selector import create_root_node
import six
from scrapy.http.request import Request
from scrapy.utils.python import to_bytes, is_listlike
from scrapy.utils.response import get_base_url
class FormRequest(Request):
@ -44,7 +45,7 @@ class FormRequest(Request):
def _get_form_url(form, url):
if url is None:
return form.action or form.base_url
return urljoin(form.base_url, form.action)
return urljoin(form.base_url, url)
@ -58,7 +59,7 @@ def _urlencode(seq, enc):
def _get_form(response, formname, formid, formnumber, formxpath):
"""Find the form element """
text = response.body_as_unicode()
root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url)
root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)

19
tests/test_http_request.py
View File

@ -801,6 +801,25 @@ class FormRequestTest(RequestTest):
self.assertEqual(fs[b'test2'], [b'val2'])
self.assertEqual(fs[b'button1'], [b''])
def test_html_base_form_action(self):
response = _buildresponse(
"""
<html>
<head>
<base href="http://b.com/">
</head>
<body>
<form action="test_form">
</form>
</body>
</html>
""",
url='http://a.com/'
)
req = self.request_class.from_response(response)
self.assertEqual(req.url, 'http://b.com/test_form')
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)
kwargs.setdefault('url', 'http://example.com')