From d42a98d3b590515bae30fb698e7aba2d7511608e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Apr 2016 00:33:25 +0200 Subject: [PATCH 1/2] Use newer w3lib.url.safe_url_string() and re-enable HTTP request tests --- requirements.txt | 2 +- scrapy/http/request/__init__.py | 6 +- tests/test_http_request.py | 285 +++++++++++++++++++++++++------- 3 files changed, 228 insertions(+), 65 deletions(-) diff --git a/requirements.txt b/requirements.txt index db042cfcf..451ab5d5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Twisted>=10.0.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.13.0 +w3lib>=1.14.1 queuelib six>=1.5.2 PyDispatcher>=2.0.5 diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6b802b36e..f48325a0f 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -8,7 +8,7 @@ import six from w3lib.url import safe_url_string from scrapy.http.headers import Headers -from scrapy.utils.python import to_native_str, to_bytes +from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax from scrapy.http.common import obsolete_setter @@ -50,8 +50,8 @@ class Request(object_ref): if not isinstance(url, six.string_types): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) - url = to_native_str(url, self.encoding) - self._url = escape_ajax(safe_url_string(url)) + s = safe_url_string(url, self.encoding) + self._url = escape_ajax(s) if ':' not in self._url: raise ValueError('Missing scheme in request url: %s' % self._url) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b81d43c41..d7216e1d2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,10 +1,13 @@ +# -*- coding: utf-8 -*- import cgi import unittest import re import six from six.moves import xmlrpc_client as xmlrpclib -from six.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse, parse_qs, unquote +if six.PY3: + from urllib.parse import unquote_to_bytes from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str @@ -86,12 +89,60 @@ class RequestTest(unittest.TestCase): r = self.request_class(url="http://www.scrapy.org/blank space") self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") - @unittest.skipUnless(six.PY2, "TODO") def test_url_encoding(self): - r1 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8") - r2 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3") - self.assertEqual(r2.url, "http://www.scrapy.org/price/%A3") + r = self.request_class(url=u"http://www.scrapy.org/price/£") + self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + + def test_url_encoding_other(self): + # encoding affects only query part of URI, not path + # path part should always be UTF-8 encoded before percent-escaping + r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8") + self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + + r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1") + self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + + def test_url_encoding_query(self): + r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ") + self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + + # should be same as above + r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + + def test_url_encoding_query_latin1(self): + # encoding is used for encoding query-string before percent-escaping; + # path is still UTF-8 encoded before percent-escaping + r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") + + def test_url_encoding_nonutf8_untouched(self): + # percent-escaping sequences that do not match valid UTF-8 sequences + # should be kept untouched (just upper-cased perhaps) + # + # See https://tools.ietf.org/html/rfc3987#section-3.2 + # + # "Conversions from URIs to IRIs MUST NOT use any character encoding + # other than UTF-8 in steps 3 and 4, even if it might be possible to + # guess from the context that another character encoding than UTF-8 was + # used in the URI. For example, the URI + # "http://www.example.org/r%E9sum%E9.html" might with some guessing be + # interpreted to contain two e-acute characters encoded as iso-8859-1. + # It must not be converted to an IRI containing these e-acute + # characters. Otherwise, in the future the IRI will be mapped to + # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different + # URI from "http://www.example.org/r%E9sum%E9.html". + r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3") + self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") + + r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + + r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3") + self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + + r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html") + self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") def test_body(self): r1 = self.request_class(url="http://www.example.com/") @@ -198,19 +249,46 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - @unittest.skipUnless(six.PY2, "TODO") - def test_default_encoding(self): + def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {'one': 'two', 'price': '\xc2\xa3 100'} + data = {b'one': b'two', b'price': b'\xc2\xa3 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two') self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') - def test_custom_encoding(self): - data = {'price': u'\xa3 100'} - r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') + def test_default_encoding_textual_data(self): + # using default encoding (utf-8) + data = {u'µ one': u'two', u'price': u'£ 100'} + r2 = self.request_class("http://www.example.com", formdata=data) + self.assertEqual(r2.method, 'POST') + self.assertEqual(r2.encoding, 'utf-8') + self.assertQueryEqual(r2.body, b'price=%C2%A3+100&%C2%B5+one=two') + self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + + def test_default_encoding_mixed_data(self): + # using default encoding (utf-8) + data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'} + r2 = self.request_class("http://www.example.com", formdata=data) + self.assertEqual(r2.method, 'POST') + self.assertEqual(r2.encoding, 'utf-8') + self.assertQueryEqual(r2.body, b'%C2%B5one=two&price%C2%A3=%C2%A3+100') + self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + + def test_custom_encoding_bytes(self): + data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} + r2 = self.request_class("http://www.example.com", formdata=data, + encoding='latin1') + self.assertEqual(r2.method, 'POST') + self.assertEqual(r2.encoding, 'latin1') + self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') + self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + + def test_custom_encoding_textual_data(self): + data = {'price': u'£ 100'} + r3 = self.request_class("http://www.example.com", formdata=data, + encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') @@ -231,15 +309,78 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html") req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) + self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) + self.assertEqual(set(fs[b'one']), {b'two', b'three'}) self.assertEqual(fs[b'test2'], [b'xxx']) self.assertEqual(fs[b'six'], [b'seven']) + def test_from_response_post_nonascii_bytes_utf8(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html") + req = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}) + + self.assertEqual(req.method, 'POST') + self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.url, "http://www.example.com/this/post.php") + fs = _qs(req, to_unicode=True) + self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) + self.assertEqual(set(fs[u'one']), {u'two', u'three'}) + self.assertEqual(fs[u'test2'], [u'xxx µ']) + self.assertEqual(fs[u'six'], [u'seven']) + + def test_from_response_post_nonascii_bytes_latin1(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + encoding='latin1', + ) + req = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}) + + self.assertEqual(req.method, 'POST') + self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.url, "http://www.example.com/this/post.php") + fs = _qs(req, to_unicode=True, encoding='latin1') + self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) + self.assertEqual(set(fs[u'one']), {u'two', u'three'}) + self.assertEqual(fs[u'test2'], [u'xxx µ']) + self.assertEqual(fs[u'six'], [u'seven']) + + def test_from_response_post_nonascii_unicode(self): + response = _buildresponse( + u"""
+ + + +
""", + url="http://www.example.com/this/list.html") + req = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}) + + self.assertEqual(req.method, 'POST') + self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.url, "http://www.example.com/this/post.php") + fs = _qs(req, to_unicode=True) + self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) + self.assertEqual(set(fs[u'one']), {u'two', u'three'}) + self.assertEqual(fs[u'test2'], [u'xxx µ']) + self.assertEqual(fs[u'six'], [u'seven']) + def test_from_response_extra_headers(self): response = _buildresponse( """
@@ -268,10 +409,10 @@ class FormRequestTest(RequestTest): self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") fs = _qs(r1) - self.assertEqual(set(fs['test']), set(['val1', 'val2'])) - self.assertEqual(set(fs['one']), set(['two', 'three'])) - self.assertEqual(fs['test2'], ['xxx']) - self.assertEqual(fs['six'], ['seven']) + self.assertEqual(set(fs[b'test']), set([b'val1', b'val2'])) + self.assertEqual(set(fs[b'one']), set([b'two', b'three'])) + self.assertEqual(fs[b'test2'], [b'xxx']) + self.assertEqual(fs[b'six'], [b'seven']) def test_from_response_override_params(self): response = _buildresponse( @@ -315,9 +456,9 @@ class FormRequestTest(RequestTest):
""") req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs['clickable1'], ['clicked1']) - self.assertFalse('i1' in fs, fs) # xpath in _get_inputs() - self.assertFalse('clickable2' in fs, fs) # xpath in _get_clickable() + self.assertEqual(fs[b'clickable1'], [b'clicked1']) + self.assertFalse(b'i1' in fs, fs) # xpath in _get_inputs() + self.assertFalse(b'clickable2' in fs, fs) # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -329,10 +470,10 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, formdata={'two': '2'}) fs = _qs(req) - self.assertEqual(fs['clickable1'], ['clicked1']) - self.assertFalse('clickable2' in fs, fs) - self.assertEqual(fs['one'], ['1']) - self.assertEqual(fs['two'], ['2']) + self.assertEqual(fs[b'clickable1'], [b'clicked1']) + self.assertFalse(b'clickable2' in fs, fs) + self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b'two'], [b'2']) def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -345,10 +486,10 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, formdata={'two': '2'}, \ clickdata={'name': 'clickable2'}) fs = _qs(req) - self.assertEqual(fs['clickable2'], ['clicked2']) - self.assertFalse('clickable1' in fs, fs) - self.assertEqual(fs['one'], ['1']) - self.assertEqual(fs['two'], ['2']) + self.assertEqual(fs[b'clickable2'], [b'clicked2']) + self.assertFalse(b'clickable1' in fs, fs) + self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b'two'], [b'2']) def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -359,7 +500,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v']}) + self.assertEqual(fs, {b'i1': [b'i1v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -371,7 +512,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v'], 'i2': ['i2v']}) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -382,11 +523,11 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, \ - clickdata={'name': 'clickable', 'value': 'clicked2'}) + clickdata={u'name': u'clickable', u'value': u'clicked2'}) fs = _qs(req) - self.assertEqual(fs['clickable'], ['clicked2']) - self.assertEqual(fs['one'], ['clicked1']) - self.assertEqual(fs['two'], ['clicked2']) + self.assertEqual(fs[b'clickable'], [b'clicked2']) + self.assertEqual(fs[b'one'], [b'clicked1']) + self.assertEqual(fs[b'two'], [b'clicked2']) def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -397,9 +538,24 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, \ - clickdata={'name': u'price in \u00a3'}) - fs = _qs(req) - self.assertTrue(fs[to_native_str(u'price in \u00a3')]) + clickdata={u'name': u'price in \u00a3'}) + fs = _qs(req, to_unicode=True) + self.assertTrue(fs[u'price in \u00a3']) + + def test_from_response_unicode_clickdata_latin1(self): + response = _buildresponse( + u"""
+ + + + +
""", + encoding='latin1') + req = self.request_class.from_response(response, \ + clickdata={u'name': u'price in \u00a5'}) + fs = _qs(req, to_unicode=True, encoding='latin1') + self.assertTrue(fs[u'price in \u00a5']) + def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -413,18 +569,18 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, formname='form2', \ - clickdata={'name': 'clickable'}) + clickdata={u'name': u'clickable'}) fs = _qs(req) - self.assertEqual(fs['clickable'], ['clicked2']) - self.assertEqual(fs['field2'], ['value2']) - self.assertFalse('field1' in fs, fs) + self.assertEqual(fs[b'clickable'], [b'clicked2']) + self.assertEqual(fs[b'field2'], [b'value2']) + self.assertFalse(b'field1' in fs, fs) def test_from_response_override_clickable(self): response = _buildresponse('''
''') req = self.request_class.from_response(response, \ formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}) fs = _qs(req) - self.assertEqual(fs['clickme'], ['two']) + self.assertEqual(fs[b'clickme'], [b'two']) def test_from_response_dont_click(self): response = _buildresponse( @@ -436,8 +592,8 @@ class FormRequestTest(RequestTest): """) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse('clickable1' in fs, fs) - self.assertFalse('clickable2' in fs, fs) + self.assertFalse(b'clickable1' in fs, fs) + self.assertFalse(b'clickable2' in fs, fs) def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -468,8 +624,8 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response(response, clickdata={'nr': 1}) fs = _qs(req) - self.assertIn('clickable2', fs) - self.assertNotIn('clickable1', fs) + self.assertIn(b'clickable2', fs) + self.assertNotIn(b'clickable1', fs) def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( @@ -490,7 +646,7 @@ class FormRequestTest(RequestTest): """""") req = self.request_class.from_response(response, formdata={'bar': 'buz'}) fs = _qs(req) - self.assertEqual(fs, {'foo': ['xxx'], 'bar': ['buz']}) + self.assertEqual(fs, {b'foo': [b'xxx'], b'bar': [b'buz']}) def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -634,7 +790,7 @@ class FormRequestTest(RequestTest):