improved url validation

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40426

scrapy/trunk/scrapy/tests/test_utils_url.py

@@ -1,5 +1,5 @@
 import unittest
-from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url
+from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, check_valid_urlencode
 
 class UrlUtilsTest(unittest.TestCase):
 
@@ -148,6 +148,13 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True),
                                           u"http://user:pass@www.example.com/do?a=1#frag")
 
+    def test_check_valid_urlencode(self):
+        self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures detail CAN43664.jpg'))
+        self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%20detail%20CAN43664.jpg'))
+
+        self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures\detail\CAN43664.jpg'))
+        self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%5Cdetail%5CCAN43664.jpg'))
+
 if __name__ == "__main__":
     unittest.main()
 

scrapy/trunk/scrapy/utils/url.py

@@ -146,3 +146,9 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False):
     if not keep_fragments:
         parts[5] = ''
     return urlparse.urlunparse(parts)
+
+def check_valid_urlencode(url):
+    """ check that the url-path is properly quoted
+    TODO should also check the parameters """
+    split_result = urlparse.urlsplit(url)
+    return split_result[2] == urllib.quote(urllib.unquote(split_result[2]))