From 08194186b4604f9bc9fc0b73bc1aa158a7fe8059 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 15 Aug 2009 19:44:37 -0300 Subject: [PATCH] improved docstring a encoding parameter of safe_url_string function. also added some unittests --- scrapy/tests/test_utils_url.py | 8 ++++++++ scrapy/utils/url.py | 22 ++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 745f748fb..05a22e8db 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -58,6 +58,14 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(safe_url_string("http://www.example.com/Brochures_&_Paint_Cards&PageSize=200"), "http://www.example.com/Brochures_&_Paint_Cards&PageSize=200") + safeurl = safe_url_string(u"http://www.example.com/\xa3", encoding='latin-1') + self.assert_(isinstance(safeurl, str)) + self.assertEqual(safeurl, "http://www.example.com/%A3") + + safeurl = safe_url_string(u"http://www.example.com/\xa3", encoding='utf-8') + self.assert_(isinstance(safeurl, str)) + self.assertEqual(safeurl, "http://www.example.com/%C2%A3") + def test_safe_download_url(self): self.assertEqual(safe_download_url('http://www.scrapy.org/../'), 'http://www.scrapy.org/') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 91dcf0b14..f0359127c 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -42,22 +42,24 @@ def urljoin_rfc(base, ref, encoding='utf-8'): return urlparse.urljoin(unicode_to_str(base, encoding), unicode_to_str(ref, encoding)) _reserved = ';/?:@&=+$|,#' # RFC 2396 (Generic Syntax) -_unreserved_marks = "-_.!~*'()" #RFC 2396 sec 2.3 +_unreserved_marks = "-_.!~*'()" # RFC 2396 sec 2.3 _safe_chars = urllib.always_safe + '%' + _reserved + _unreserved_marks -def safe_url_string(url, use_encoding='utf8'): - """Convert a unicode object (using 'use_encoding' as the encoding), or an already - encoded string into a legal URL. +def safe_url_string(url, encoding='utf8'): + """Convert the given url into a legal URL by escaping unsafe characters + according to RFC-2396. - Illegal characters are escaped (RFC-3986) + If a unicode url is given, it is first converted to str using the given + encoding (which defaults to 'utf-8'). When passing a encoding, you should + use the encoding of the original page (the page from which the url was + extracted from). - It is safe to call this function multiple times. + Calling this function on an already "safe" url will return the url + unmodified. - The use_encoding argument is the encoding to use to determine the numerical - values in the escaping. For urls on html pages, you should use the original - encoding of that page. + Always returns a str. """ - s = unicode_to_str(url, use_encoding) + s = unicode_to_str(url, encoding) return urllib.quote(s, _safe_chars)