RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2548 from scrapy/formrequest-whitespaces 

[MRG+1] FormRequest: handle whitespaces in action attribute properly
This commit is contained in:
Daniel Graña 2017-02-20 11:09:50 -03:00 committed by GitHub
parent 4a93be4ad8 074caf434e
commit 322fd68e4c
2 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/http/request/form.py
View File

@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import six
from six.moves.urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
import six
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.utils.python import to_bytes, is_listlike
from scrapy.utils.response import get_base_url
@ -51,7 +54,10 @@ class FormRequest(Request):
def _get_form_url(form, url):
if url is None:
return urljoin(form.base_url, form.action)
action = form.get('action')
if action is None:
return form.base_url
return urljoin(form.base_url, strip_html5_whitespace(action))
return urljoin(form.base_url, url)

10
tests/test_http_request.py
View File

@ -556,7 +556,6 @@ class FormRequestTest(RequestTest):
fs = _qs(req, to_unicode=True, encoding='latin1')
self.assertTrue(fs[u'price in \u00a5'])
def test_from_response_multiple_forms_clickdata(self):
response = _buildresponse(
"""<form name="form1">
@ -989,7 +988,7 @@ class FormRequestTest(RequestTest):
"""
<html>
<head>
<base href="http://b.com/">
<base href=" http://b.com/">
</head>
<body>
<form action="test_form">
@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest):
req = self.request_class.from_response(response)
self.assertEqual(req.url, 'http://b.com/test_form')
def test_spaces_in_action(self):
resp = _buildresponse('<body><form action=" path\n"></form></body>')
req = self.request_class.from_response(resp)
self.assertEqual(req.url, 'http://example.com/path')
def test_from_response_css(self):
response = _buildresponse(
"""<form action="post.php" method="POST">
@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest):
self.assertRaises(ValueError, self.request_class.from_response,
response, formcss="input[name='abc']")
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)
kwargs.setdefault('url', 'http://example.com')
kwargs.setdefault('encoding', 'utf-8')
return HtmlResponse(**kwargs)
def _qs(req, encoding='utf-8', to_unicode=False):
if req.method == 'POST':
qs = req.body