From fe49bc2011a233c22095f6eb22c7fc907fbc1924 Mon Sep 17 00:00:00 2001 From: samus_ Date: Wed, 26 Nov 2008 12:05:31 +0000 Subject: [PATCH] improved check --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40428 --- scrapy/trunk/scrapy/tests/test_utils_url.py | 12 +++++++++--- scrapy/trunk/scrapy/utils/url.py | 10 +++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index da30f501d..9d8709511 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -149,12 +149,18 @@ class UrlUtilsTest(unittest.TestCase): u"http://user:pass@www.example.com/do?a=1#frag") def test_check_valid_urlencode(self): - self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures detail CAN43664.jpg')) - self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%20detail%20CAN43664.jpg')) - self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures\detail\CAN43664.jpg')) self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%5Cdetail%5CCAN43664.jpg')) + self.assertFalse(check_valid_urlencode('http://www.example.com/pictures detail CAN43664.jpg')) + self.assertTrue(check_valid_urlencode('http://www.example.com/pictures+detail%20CAN43664.jpg')) + + self.assertFalse(check_valid_urlencode('http://www.example.com/?q=foo bar&q2=foo2 bar2')) + self.assertTrue(check_valid_urlencode('http://www.example.com/?q=foo+bar&q2=foo2%20bar2')) + + self.assertFalse(check_valid_urlencode('http://www.example.com/product,little:london$set%')) + self.assertTrue(check_valid_urlencode('http://www.example.com/product%2Clittle%3Alondon%24set%25')) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index f75348c6c..b989f1941 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -148,7 +148,11 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): return urlparse.urlunparse(parts) def check_valid_urlencode(url): - """ check that the url-path is properly quoted - TODO should also check the parameters """ + """ check that the url-path and arguments are properly quoted """ + def check_str(s): + return all((s_plus == urllib.quote(urllib.unquote(s_plus)) for s_plus in s.split('+'))) + def check_param(p): + return all((check_str(s) for s in p.split('=', 1))) + split_result = urlparse.urlsplit(url) - return split_result[2] == urllib.quote(urllib.unquote(split_result[2])) + return check_str(split_result[2]) and all((check_param(p) for p in split_result[3].split('&')))