From b65c87cb67176bc4fad650aba1ee55b8580aa1da Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 20 Aug 2009 14:23:49 -0300 Subject: [PATCH] removed unused module: scrapy.utils.c14n --- scrapy/tests/test_c14nurls.py | 28 --------- scrapy/utils/c14n.py | 110 ---------------------------------- 2 files changed, 138 deletions(-) delete mode 100644 scrapy/tests/test_c14nurls.py delete mode 100644 scrapy/utils/c14n.py diff --git a/scrapy/tests/test_c14nurls.py b/scrapy/tests/test_c14nurls.py deleted file mode 100644 index e0a8e8166..000000000 --- a/scrapy/tests/test_c14nurls.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -from scrapy.utils.c14n import canonicalize - -class C14nTest(unittest.TestCase): - """ c14n comparison functions """ - - def test_canonicalize(self): - """Test URL canonicalization function""" - urls = [ - ('http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2', - 'http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2'), - - - ('http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages', - 'http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages'), - - ('http://www.test-nice-url.com/index.html', - 'http://www.test-nice-url.com/index.html'), - - ('http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704', - 'http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704'), - ] - for origurl, c14nurl in urls: - self.assertEqual(canonicalize(origurl), c14nurl) - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/utils/c14n.py b/scrapy/utils/c14n.py deleted file mode 100644 index 88cdd16df..000000000 --- a/scrapy/utils/c14n.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Canonicalization routine for URLs -""" -import re -import urlparse - -_hexmatcher = re.compile("\%[0-9A-Fa-f]{2}") - -def get_canonized_port(port): - if port and port != 80: - cport = ":" + str(port) - else: - cport = "" - return cport - -def create_params_dict(params): - param_dict = {} - for param in params.split("&"): - nvpair = param.split("=") - if len(nvpair) >= 2: - param_dict[nvpair[0]] = '='.join(nvpair[1:]) - else: - param_dict[nvpair[0]] = None - return param_dict - -def create_unique_params_list(params): - """ - This function build a query string from unique params only. - Unique params are params with unique key=>value pair. So param=qwe and param=123 - are diffrent and a query string will be param=qwe¶m=123. - If input params will contain pairs like this param=123 and param=123 a query - string will be just param=123 - We have to do this because some ASP sites uses the same parameters names in the URLs, - but with diffrent values. Such queries are interpreted as arrays in the ASP framework. - """ - param_list = [] - if params: - for param in params.split("&"): - nvpair = param.split("=", 1) - if len(nvpair) >= 2: - param_list.append(nvpair[0] + '=' + nvpair[1]) - else: - param_list.append(nvpair[0] + '=') - if param_list: - p_list = [] - # select only unique pairs, it is possible to use set() for this but change order of pairs... - for param in param_list: - if param not in p_list: - p_list.append(param) - return '&'.join(p_list) - return '' - -def convert_escaped(string_to_convert): - hexvalues = _hexmatcher.findall(string_to_convert) - for hexvalue in hexvalues: - string_to_convert = string_to_convert.replace( - hexvalue, chr(int(hexvalue[1:len(hexvalue)],16))) - return string_to_convert - -def remove_dirs(path): - dirs = path.split("/") - dirs.reverse() - stripped_dirs = [] - ellipsis = 0 - for dir in dirs: - if dir == ".." : - ellipsis += 1 - elif dir == "." : - pass - else: - if ellipsis > 0 : - ellipsis -= 1 - else: - stripped_dirs.append(dir) - stripped_dirs.reverse() - return "/".join(stripped_dirs) - -def canonicalize(url): - """ - This method c14ns the url we are passed by doing the following - 1) lowercasing the hostname and scheme - 2) removing the port if it is 80 - 3) making the query params unique and alphabetized - 4) removing url encoding - 5) Removing any ./ - 6) Removing and ../ and their preceding directories - """ - - # Remove the fragment marker, even if it's url encoded - defragmented_url = urlparse.urldefrag(url)[0] - - # This automatically converts the scheme and hostname to lower case - parsed = urlparse.urlparse(defragmented_url) - - # Get the port in the correct format - cport = get_canonized_port(parsed.port) - - query = create_unique_params_list(parsed.query) - - # Remove the url encoded params from the path and query string - query = convert_escaped(query) - if query : - query = "?" + query - path = convert_escaped(parsed.path) - - # Remove the ../ and the previous directories and remove the ./ - path = remove_dirs(path) - - # Now put it all back together - return "%s://%s%s%s%s" % (parsed.scheme, parsed.hostname, cport, path, query)