@@ -43,44 +104,54 @@ class AdaptorsTestCase(unittest.TestCase):
u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/imgs/lala4.jpg'])
self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')),
[u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
+
def test_to_unicode(self):
- self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
+ self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'lulu\xc3\xb1', 1, '\xc3\xa1\xc3\xa9']),
[u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
+
def test_regex(self):
adaptor = adaptors.Regex(regex=r'href="(.*?)"')
self.assertEqual(adaptor(['
dsa',
'
href="lelelel.net"']),
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
+
def test_unquote_all(self):
self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&'])
+
def test_unquote(self):
self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
+
def test_remove_tags(self):
test_data = ['
adsaas
', '
']
self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
+
def test_remove_root(self):
self.assertEqual(adaptors.remove_root(['
']),
['lallaa
dsfsdfdspepepep
'])
+
def test_remove_multispaces(self):
self.assertEqual(adaptors.clean_spaces([' hello, whats up?', 'testing testingtesting testing']),
[' hello, whats up?', 'testing testingtesting testing'])
+
def test_strip(self):
self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
['hi there, sweety ;D', 'I CAN HAZ TEST??'])
self.assertEqual(adaptors.strip(' hello there, this is my test '),
'hello there, this is my test')
+
def test_drop_empty_elements(self):
self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']),
[1, 2, 5, 6, 'hi'])
+
def test_delist(self):
self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
diff --git a/scrapy/trunk/scrapy/tests/test_defaultencoding.py b/scrapy/trunk/scrapy/tests/test_defaultencoding.py
deleted file mode 100644
index 36c5b782d..000000000
--- a/scrapy/trunk/scrapy/tests/test_defaultencoding.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import sys
-from unittest import TestCase, main
-
-class DefaultEncodingTest(TestCase):
- def test_defaultencoding(self):
- self.assertEqual(sys.getdefaultencoding(), 'utf-8')
-
-if __name__ == "__main__":
- main()
diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py
index d585dcc06..2ff0d5f33 100644
--- a/scrapy/trunk/scrapy/utils/iterators.py
+++ b/scrapy/trunk/scrapy/utils/iterators.py
@@ -3,16 +3,16 @@ import re, csv
from scrapy.xpath import XmlXPathSelector
from scrapy.http import Response
from scrapy import log
-from scrapy.utils.python import re_rsearch
+from scrapy.utils.python import re_rsearch, str_to_unicode, unicode_to_str
-def _normalize_input(obj):
+def _normalize_input(obj, unicode=True):
assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
if isinstance(obj, Response):
- return obj.body.to_unicode()
+ return obj.body.to_unicode() if unicode else obj.body.to_string()
elif isinstance(obj, str):
- return obj.decode('utf-8')
+ return obj.decode('utf-8') if unicode else obj
else:
- return obj
+ return obj if unicode else obj.encode('utf-8')
def xmliter(obj, nodename):
"""Return a iterator of XPathSelector's over all nodes of a XML document,
@@ -51,9 +51,9 @@ def csviter(obj, delimiter=None, headers=None):
for the returned dictionaries, if not the first row is used.
"""
def _getrow(csv_r):
- return [field.decode() for field in csv_r.next()]
+ return [str_to_unicode(field) for field in csv_r.next()]
- lines = _normalize_input(obj).splitlines(True)
+ lines = _normalize_input(obj, unicode=False).splitlines(True)
if delimiter:
csv_r = csv.reader(lines, delimiter=delimiter)
else:
diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py
index e16e76c06..8963d90d5 100644
--- a/scrapy/trunk/scrapy/utils/url.py
+++ b/scrapy/trunk/scrapy/utils/url.py
@@ -9,6 +9,8 @@ import urllib
import posixpath
import cgi
+from scrapy.utils.python import unicode_to_str
+
def url_is_from_any_domain(url, domains):
"""Return True if the url belongs to the given domain"""
host = urlparse.urlparse(url).hostname
@@ -141,7 +143,7 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False):
For examples see the tests in scrapy.tests.test_utils_url
"""
- url = url.encode('utf-8')
+ url = unicode_to_str(url)
parts = list(urlparse.urlparse(url))
keyvals = cgi.parse_qsl(parts[4], keep_blank_values)
keyvals.sort()
diff --git a/scrapy/trunk/scrapy/xpath/selector.py b/scrapy/trunk/scrapy/xpath/selector.py
index 7ddd108dc..f614f9f5f 100644
--- a/scrapy/trunk/scrapy/xpath/selector.py
+++ b/scrapy/trunk/scrapy/xpath/selector.py
@@ -65,13 +65,13 @@ class XPathSelector(object):
text = unicode(data, 'utf-8', errors='ignore') if data else u''
elif isinstance(self.xmlNode, libxml2.xmlAttr):
# serialization doesn't work sometimes for xmlAttr types
- text = unicode(self.xmlNode.content, errors='ignore')
+ text = unicode(self.xmlNode.content, 'utf-8', errors='ignore')
else:
data = self.xmlNode.serialize('utf-8')
text = unicode(data, 'utf-8', errors='ignore') if data else u''
else:
try:
- text = unicode(self.xmlNode, errors='ignore')
+ text = unicode(self.xmlNode, 'utf-8', errors='ignore')
except TypeError: # catched when self.xmlNode is a float - see tests
text = unicode(self.xmlNode)
return text
@@ -79,7 +79,7 @@ class XPathSelector(object):
def extract_unquoted(self):
"""Get unescaped contents from the text node (no entities, no CDATA)"""
if self.x('self::text()'):
- return unicode(self.xmlNode.getContent(), errors='ignore')
+ return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
else:
return u''