From f6a0ac8325fbd442875b54dc3f5a12f5454123be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 30 Dec 2013 15:13:18 -0200 Subject: [PATCH] support overriding url in FormRequest.from_response --- scrapy/http/request/form.py | 10 +++++++--- scrapy/tests/test_http_request.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 5c639a605..69088c48b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more covenient class See documentation in docs/topics/request-response.rst """ -import urllib +import urllib, urlparse import lxml.html from scrapy.http.request import Request from scrapy.utils.python import unicode_to_str @@ -35,10 +35,14 @@ class FormRequest(Request): kwargs.setdefault('encoding', response.encoding) form = _get_form(response, formname, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) - url = form.action or form.base_url + url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) - return cls(url, method=method, formdata=formdata, **kwargs) + return cls(url=url, method=method, formdata=formdata, **kwargs) +def _get_form_url(form, url): + if url is None: + return form.action or form.base_url + return urlparse.urljoin(form.base_url, url) def _urlencode(seq, enc): values = [(unicode_to_str(k, enc), unicode_to_str(v, enc)) diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index a1cc162ff..069b1cd04 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -282,6 +282,18 @@ class FormRequestTest(RequestTest): request = FormRequest.from_response(response, method='POST') self.assertEqual(request.method, 'POST') + def test_from_response_override_url(self): + response = _buildresponse( + ''' +
+ ''') + request = FormRequest.from_response(response) + self.assertEqual(request.url, 'http://example.com/app') + request = FormRequest.from_response(response, url='http://foo.bar/absolute') + self.assertEqual(request.url, 'http://foo.bar/absolute') + request = FormRequest.from_response(response, url='/relative') + self.assertEqual(request.url, 'http://example.com/relative') + def test_from_response_submit_first_clickable(self): response = _buildresponse( """