From bddfbee0d497366885bc3210d3cbf5c4ef842282 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Fri, 14 Nov 2008 13:36:18 +0000 Subject: [PATCH] Fixed some unicode issues in adaptors, improved docstrings and a test --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40381 --- .../trunk/scrapy/contrib/adaptors/markup.py | 8 +++- scrapy/trunk/scrapy/contrib/adaptors/misc.py | 39 +++++++++++++------ scrapy/trunk/scrapy/tests/test_adaptors.py | 2 +- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/adaptors/markup.py b/scrapy/trunk/scrapy/contrib/adaptors/markup.py index b912f73e1..9b1d048f3 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/markup.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/markup.py @@ -3,8 +3,12 @@ from scrapy.utils.markup import replace_tags, remove_entities def remove_tags(value): """ + Removes any tags found in each of the provided list's string. + E.g: + >> remove_tags(['my header', 'my body']) + [u'my header', u'my body'] Input: iterable with strings - Output: list of strings + Output: list of unicodes """ return [ replace_tags(v) for v in value ] @@ -18,7 +22,7 @@ def remove_root(value): m = _remove_root_re.search(value) if m: value = m.group(1) - return value + return unicode(value) return [ _remove_root(v) for v in value ] class Unquote(object): diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py index 43deca741..71b5d04e4 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import re from scrapy.xpath.selector import XPathSelector, XPathSelectorList from scrapy.utils.url import canonicalize_url @@ -8,8 +10,11 @@ def to_unicode(value): """ Receives a list of strings, converts it to unicode, and returns a new list. + E.g: + >> to_unicode(['it costs 20€, or 30£']) + [u'it costs 20\u20ac, or 30\xa3'] - Input: iterable with strings + Input: iterable of strings Output: list of unicodes """ if hasattr(value, '__iter__'): @@ -19,29 +24,38 @@ def to_unicode(value): def clean_spaces(value): """ - Converts multispaces into single spaces. - E.g. "Hello sir" would turn into "Hello sir". + Converts multispaces into single spaces for each string + in the provided iterable. + E.g: + >> clean_spaces(['Hello sir']) + [u'Hello sir'] - Input: list of unicodes + Input: iterable of unicodes Output: list of unicodes """ _clean_spaces_re = re.compile("\s+", re.U) - return [ _clean_spaces_re.sub(' ', v) for v in value ] + return [ _clean_spaces_re.sub(' ', v.decode('utf-8')) for v in value ] def strip_list(value): """ Removes any spaces at both the start and the ending of each string in the provided list. + E.g: + >> strip_list([' hi ', 'buddies ']) + [u'hi', u'buddies'] - Input: list of unicodes + Input: iterable of unicodes Output: list of unicodes """ - return [ v.strip() for v in value ] + return [ unicode(v.strip()) for v in value ] def drop_empty(value): """ Removes any index that evaluates to None from the provided iterable. + E.g: + >> drop_empty([0, 'this', None, 'is', False, 'an example']) + ['this', 'is', 'an example'] Input: iterable Output: list @@ -50,22 +64,25 @@ def drop_empty(value): def canonicalize_urls(value): """ - Tries to canonicalize each url in the list you provide. + Canonicalizes each url in the list you provide. To see what this implies, check out canonicalize_url's docstring, at scrapy.utils.url.py - Input: list of unicodes(urls) + Input: iterable of unicodes(urls) Output: list of unicodes(urls) """ if hasattr(value, '__iter__'): return [canonicalize_url(url) for url in value] elif isinstance(value, basestring): return canonicalize_url(value) - return '' + return u'' class Delist(object): """ - Input: iterable with strings + Joins a list with the specified delimiter + in the adaptor's constructor. + + Input: iterable of strings Output: unicode """ def __init__(self, delimiter=' '): diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py index 63b4d3a1f..881c40a06 100644 --- a/scrapy/trunk/scrapy/tests/test_adaptors.py +++ b/scrapy/trunk/scrapy/tests/test_adaptors.py @@ -74,7 +74,7 @@ class AdaptorsTestCase(unittest.TestCase): ['hi there, sweety ;D', 'I CAN HAZ TEST??']) def test_drop_empty_elements(self): - self.assertEqual(adaptors.drop_empty([1, 2, None, 5, None, 6, None, 'hi']), + self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']), [1, 2, 5, 6, 'hi']) def test_delist(self):