diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index 9d8709511..2b0fcb660 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -158,8 +158,9 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(check_valid_urlencode('http://www.example.com/?q=foo bar&q2=foo2 bar2')) self.assertTrue(check_valid_urlencode('http://www.example.com/?q=foo+bar&q2=foo2%20bar2')) - self.assertFalse(check_valid_urlencode('http://www.example.com/product,little:london$set%')) - self.assertTrue(check_valid_urlencode('http://www.example.com/product%2Clittle%3Alondon%24set%25')) + self.assertFalse(check_valid_urlencode('http://www.example.com/.,:;!@$%^*()_-[]{}|')) + self.assertTrue(check_valid_urlencode('http://www.example.com/.,:;!@%24%25%5E*()_-%5B%5D%7B%7D%7C')) + self.assertTrue(check_valid_urlencode('http://www.example.com/.%2C%3A%3B%21%40%24%25%5E%2A%28%29_-%5B%5D%7B%7D%7C')) if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index b989f1941..5f07283f7 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -150,9 +150,11 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): def check_valid_urlencode(url): """ check that the url-path and arguments are properly quoted """ def check_str(s): - return all((s_plus == urllib.quote(urllib.unquote(s_plus)) for s_plus in s.split('+'))) + for ignore_char in ',:;!@*()': + s = s.replace(ignore_char, '') + return all(s_plus == urllib.quote(urllib.unquote(s_plus)) for s_plus in s.split('+')) def check_param(p): - return all((check_str(s) for s in p.split('=', 1))) + return all(check_str(s) for s in p.split('=', 1)) split_result = urlparse.urlsplit(url) - return check_str(split_result[2]) and all((check_param(p) for p in split_result[3].split('&'))) + return check_str(split_result.path) and all(check_param(p) for p in split_result.query.split('&'))