From 34355048c3221e6dc314b17c94cdb0a0cb41eef6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 1 Sep 2008 01:19:32 +0000 Subject: [PATCH] some functions were added to scrapy.utils.url without following our policies for adding tests (to scrapy.tests) and documentation (as docstrings). fixed that. --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40186 --- scrapy/trunk/scrapy/tests/test_utils_url.py | 33 ++++++++++++- scrapy/trunk/scrapy/utils/url.py | 51 ++++----------------- 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index 9b91c18da..cb5787868 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -1,5 +1,5 @@ import unittest -from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url +from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner class UrlUtilsTest(unittest.TestCase): @@ -43,6 +43,37 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(safe_download_url('http://www.scrapy.org/dir/'), 'http://www.scrapy.org/dir/') + def test_url_query_parameter(self): + self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "id"), + '200') + self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault"), + 'mydefault') + self.assertEqual(url_query_parameter("product.html?id=", "id"), + None) + self.assertEqual(url_query_parameter("product.html?id=", "id", keep_blank_values=1), + '') + + def test_add_or_replace_parameter(self): + url = 'http://domain/test' + self.assertEqual(add_or_replace_parameter(url, 'arg', 'v'), + 'http://domain/test?arg=v') + url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3' + self.assertEqual(add_or_replace_parameter(url, 'arg4', 'v4'), + 'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4') + self.assertEqual(add_or_replace_parameter(url, 'arg3', 'nv3'), + 'http://domain/test?arg1=v1&arg2=v2&arg3=nv3') + url = 'http://domain/test?arg1=v1' + self.assertEqual(add_or_replace_parameter(url, 'arg2', 'v2', sep=';'), + 'http://domain/test?arg1=v1;arg2=v2') + self.assertEqual(add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20'), + 'http://domain/moreInfo.asp?prodID=20') + + def test_url_query_cleaner(self): + self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'), + 'product.html?id=200') + self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']), + 'product.html?id=200&name=wired') + if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index b976d26f7..1a4a146e1 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -78,35 +78,16 @@ def safe_download_url(url): path = '/' return urlparse.urlunsplit((scheme, netloc, path, query, '')) - def is_url(text): return text.partition("://")[0] in ('file', 'http', 'https') def url_query_parameter(url, parameter, default=None, keep_blank_values=0): - """ Return the given query parameter in the url. - For example: - >>> url_query_parameter("product.html?id=200&foo=bar", "id") - '200' - >>> url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault") - 'mydefault' - >>> url_query_parameter("product.html?id=", "id") - >>> url_query_parameter("product.html?id=", "id", keep_blank_values=1) - '' - """ + """Return the value of a url parameter, given the url and parameter name""" queryparams = cgi.parse_qs(urlparse.urlsplit(str(url))[3], keep_blank_values=keep_blank_values) return queryparams.get(parameter, [default])[0] -def url_query_cleaner(url, parameterlist=None, sep='&', kvsep='='): - """ Return the given url with given query parameters. - >>> url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id') - 'product.html?id=200' - >>> url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']) - 'product.html?id=200&name=wired' - """ - parameterlist = parameterlist or [] - if not isinstance(parameterlist, (list, tuple)): - parameterlist = [parameterlist] - +def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='): + """Clean url arguments leaving only those passed in the parameterlist""" try: base, query = url.split('?', 1) parameters = [pair.split(kvsep, 1) for pair in query.split(sep)] @@ -127,29 +108,15 @@ def url_query_cleaner(url, parameterlist=None, sep='&', kvsep='='): query = sep.join([kvsep.join(pair) for pair in querylist if pair[0] in parameterlist]) return '?'.join([base, query]) -def _has_querystring(url): - _, _, _, query, _ = urlparse.urlsplit(url) - return bool(query) - def add_or_replace_parameter(url, name, new_value, sep='&'): - """ - >>> url = 'http://domain/test' - >>> add_or_replace_parameter(url, 'arg', 'v') - 'http://domain/test?arg=v' - >>> url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3' - >>> add_or_replace_parameter(url, 'arg4', 'v4') - 'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4' - >>> add_or_replace_parameter(url, 'arg3', 'nv3') - 'http://domain/test?arg1=v1&arg2=v2&arg3=nv3' - >>> url = 'http://domain/test?arg1=v1' - >>> add_or_replace_parameter(url, 'arg2', 'v2', sep=';') - 'http://domain/test?arg1=v1;arg2=v2' - >>> add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20') - 'http://domain/moreInfo.asp?prodID=20' - """ + """Add or remove a parameter to a given url""" + def has_querystring(url): + _, _, _, query, _ = urlparse.urlsplit(url) + return bool(query) + parameter = url_query_parameter(url, name, keep_blank_values=1) if parameter is None: - if _has_querystring(url): + if has_querystring(url): next_url = url + sep + name + '=' + new_value else: next_url = url + '?' + name + '=' + new_value