RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #507 from dangra/fromresponse_override_url 

[MRG] support overriding url in FormRequest.from_response
This commit is contained in:
Pablo Hoffman 2013-12-31 12:01:13 -08:00
parent fa82cc6794 f6a0ac8325
commit e2f091801c
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/http/request/form.py
View File

@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more covenient class
See documentation in docs/topics/request-response.rst
"""
import urllib
import urllib, urlparse
import lxml.html
from scrapy.http.request import Request
from scrapy.utils.python import unicode_to_str
@ -35,10 +35,14 @@ class FormRequest(Request):
kwargs.setdefault('encoding', response.encoding)
form = _get_form(response, formname, formnumber, formxpath)
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
url = form.action or form.base_url
url = _get_form_url(form, kwargs.pop('url', None))
method = kwargs.pop('method', form.method)
return cls(url, method=method, formdata=formdata, **kwargs)
return cls(url=url, method=method, formdata=formdata, **kwargs)
def _get_form_url(form, url):
if url is None:
return form.action or form.base_url
return urlparse.urljoin(form.base_url, url)
def _urlencode(seq, enc):
values = [(unicode_to_str(k, enc), unicode_to_str(v, enc))

12
scrapy/tests/test_http_request.py
View File

@ -282,6 +282,18 @@ class FormRequestTest(RequestTest):
request = FormRequest.from_response(response, method='POST')
self.assertEqual(request.method, 'POST')
def test_from_response_override_url(self):
response = _buildresponse(
'''<html><body>
<form action="/app"></form>
</body></html>''')
request = FormRequest.from_response(response)
self.assertEqual(request.url, 'http://example.com/app')
request = FormRequest.from_response(response, url='http://foo.bar/absolute')
self.assertEqual(request.url, 'http://foo.bar/absolute')
request = FormRequest.from_response(response, url='/relative')
self.assertEqual(request.url, 'http://example.com/relative')
def test_from_response_submit_first_clickable(self):
response = _buildresponse(
"""<form action="get.php" method="GET">