From 142cd72faee721774f5983f388b9766d9159fe7e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 27 Nov 2008 15:21:41 +0000 Subject: [PATCH] removed function check_valid_urlencode from scrapy.utils.url (it didn't fit there because it was too custom) --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40445 --- scrapy/trunk/scrapy/tests/test_utils_url.py | 16 +--------------- scrapy/trunk/scrapy/utils/url.py | 13 ------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index e68136a6a..510ccf0a5 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -1,5 +1,5 @@ import unittest -from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, check_valid_urlencode +from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url class UrlUtilsTest(unittest.TestCase): @@ -159,20 +159,6 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'), 'http://www.example.com/caf%E9-con-leche.htm') - def test_check_valid_urlencode(self): - self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures\detail\CAN43664.jpg')) - self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%5Cdetail%5CCAN43664.jpg')) - - self.assertFalse(check_valid_urlencode('http://www.example.com/pictures detail CAN43664.jpg')) - self.assertTrue(check_valid_urlencode('http://www.example.com/pictures+detail%20CAN43664.jpg')) - - self.assertFalse(check_valid_urlencode('http://www.example.com/?q=foo bar&q2=foo2 bar2')) - self.assertTrue(check_valid_urlencode('http://www.example.com/?q=foo+bar&q2=foo2%20bar2')) - - self.assertFalse(check_valid_urlencode('http://www.example.com/.,:;!@$%^*()_-[]{}|')) - self.assertTrue(check_valid_urlencode('http://www.example.com/.,:;!@%24%25%5E*()_-%5B%5D%7B%7D%7C')) - self.assertTrue(check_valid_urlencode('http://www.example.com/.%2C%3A%3B%21%40%24%25%5E%2A%28%29_-%5B%5D%7B%7D%7C')) - if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index 8d8fd08f1..e16e76c06 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -150,16 +150,3 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): if not keep_fragments: parts[5] = '' return urlparse.urlunparse(parts) - -def check_valid_urlencode(url): - """ check that the url-path and arguments are properly quoted """ - def check_str(s): - for ignore_char in ',:;!@*()': - s = s.replace(ignore_char, '') - return all(s_plus == urllib.quote(urllib.unquote(s_plus)) for s_plus in s.split('+')) - def check_param(p): - return all(check_str(s) for s in p.split('=', 1)) - - url = url.encode('utf-8') - split_result = urlparse.urlsplit(url) - return check_str(split_result.path) and all(check_param(p) for p in split_result.query.split('&'))