RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some functions were added to scrapy.utils.url without following our policies for adding tests (to scrapy.tests) and documentation (as docstrings). fixed that. 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40186
This commit is contained in:
Pablo Hoffman 2008-09-01 01:19:32 +00:00
parent f3013bb9ad
commit 34355048c3
2 changed files with 41 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
scrapy/trunk/scrapy/tests/test_utils_url.py
View File

@ -1,5 +1,5 @@
import unittest
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner
class UrlUtilsTest(unittest.TestCase):
@ -43,6 +43,37 @@ class UrlUtilsTest(unittest.TestCase):
self.assertEqual(safe_download_url('http://www.scrapy.org/dir/'),
'http://www.scrapy.org/dir/')
def test_url_query_parameter(self):
self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "id"),
'200')
self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault"),
'mydefault')
self.assertEqual(url_query_parameter("product.html?id=", "id"),
None)
self.assertEqual(url_query_parameter("product.html?id=", "id", keep_blank_values=1),
'')
def test_add_or_replace_parameter(self):
url = 'http://domain/test'
self.assertEqual(add_or_replace_parameter(url, 'arg', 'v'),
'http://domain/test?arg=v')
url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3'
self.assertEqual(add_or_replace_parameter(url, 'arg4', 'v4'),
'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4')
self.assertEqual(add_or_replace_parameter(url, 'arg3', 'nv3'),
'http://domain/test?arg1=v1&arg2=v2&arg3=nv3')
url = 'http://domain/test?arg1=v1'
self.assertEqual(add_or_replace_parameter(url, 'arg2', 'v2', sep=';'),
'http://domain/test?arg1=v1;arg2=v2')
self.assertEqual(add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20'),
'http://domain/moreInfo.asp?prodID=20')
def test_url_query_cleaner(self):
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'),
'product.html?id=200')
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']),
'product.html?id=200&name=wired')
if __name__ == "__main__":
unittest.main()

51
scrapy/trunk/scrapy/utils/url.py
View File

@ -78,35 +78,16 @@ def safe_download_url(url):
path = '/'
return urlparse.urlunsplit((scheme, netloc, path, query, ''))
def is_url(text):
return text.partition("://")[0] in ('file', 'http', 'https')
def url_query_parameter(url, parameter, default=None, keep_blank_values=0):
""" Return the given query parameter in the url.
For example:
>>> url_query_parameter("product.html?id=200&foo=bar", "id")
'200'
>>> url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault")
'mydefault'
>>> url_query_parameter("product.html?id=", "id")
>>> url_query_parameter("product.html?id=", "id", keep_blank_values=1)
''
"""
"""Return the value of a url parameter, given the url and parameter name"""
queryparams = cgi.parse_qs(urlparse.urlsplit(str(url))[3], keep_blank_values=keep_blank_values)
return queryparams.get(parameter, [default])[0]
def url_query_cleaner(url, parameterlist=None, sep='&', kvsep='='):
""" Return the given url with given query parameters.
>>> url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id')
'product.html?id=200'
>>> url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name'])
'product.html?id=200&name=wired'
"""
parameterlist = parameterlist or []
if not isinstance(parameterlist, (list, tuple)):
parameterlist = [parameterlist]
def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='):
"""Clean url arguments leaving only those passed in the parameterlist"""
try:
base, query = url.split('?', 1)
parameters = [pair.split(kvsep, 1) for pair in query.split(sep)]
@ -127,29 +108,15 @@ def url_query_cleaner(url, parameterlist=None, sep='&', kvsep='='):
query = sep.join([kvsep.join(pair) for pair in querylist if pair[0] in parameterlist])
return '?'.join([base, query])
def _has_querystring(url):
_, _, _, query, _ = urlparse.urlsplit(url)
return bool(query)
def add_or_replace_parameter(url, name, new_value, sep='&'):
"""
>>> url = 'http://domain/test'
>>> add_or_replace_parameter(url, 'arg', 'v')
'http://domain/test?arg=v'
>>> url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3'
>>> add_or_replace_parameter(url, 'arg4', 'v4')
'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4'
>>> add_or_replace_parameter(url, 'arg3', 'nv3')
'http://domain/test?arg1=v1&arg2=v2&arg3=nv3'
>>> url = 'http://domain/test?arg1=v1'
>>> add_or_replace_parameter(url, 'arg2', 'v2', sep=';')
'http://domain/test?arg1=v1;arg2=v2'
>>> add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20')
'http://domain/moreInfo.asp?prodID=20'
"""
"""Add or remove a parameter to a given url"""
def has_querystring(url):
_, _, _, query, _ = urlparse.urlsplit(url)
return bool(query)
parameter = url_query_parameter(url, name, keep_blank_values=1)
if parameter is None:
if _has_querystring(url):
if has_querystring(url):
next_url = url + sep + name + '=' + new_value
else:
next_url = url + '?' + name + '=' + new_value