From 664a72490fc738f38916af59daf65541261d7e4f Mon Sep 17 00:00:00 2001 From: samus_ Date: Tue, 25 Nov 2008 18:49:17 +0000 Subject: [PATCH] improved url validation --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40426 --- scrapy/trunk/scrapy/tests/test_utils_url.py | 9 ++++++++- scrapy/trunk/scrapy/utils/url.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index 39687c313..da30f501d 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -1,5 +1,5 @@ import unittest -from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url +from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, check_valid_urlencode class UrlUtilsTest(unittest.TestCase): @@ -148,6 +148,13 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True), u"http://user:pass@www.example.com/do?a=1#frag") + def test_check_valid_urlencode(self): + self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures detail CAN43664.jpg')) + self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%20detail%20CAN43664.jpg')) + + self.assertFalse(check_valid_urlencode(r'http://www.example.com/pictures\detail\CAN43664.jpg')) + self.assertTrue(check_valid_urlencode('http://www.example.com/pictures%5Cdetail%5CCAN43664.jpg')) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index 131875a54..f75348c6c 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -146,3 +146,9 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): if not keep_fragments: parts[5] = '' return urlparse.urlunparse(parts) + +def check_valid_urlencode(url): + """ check that the url-path is properly quoted + TODO should also check the parameters """ + split_result = urlparse.urlsplit(url) + return split_result[2] == urllib.quote(urllib.unquote(split_result[2]))