From 5eb7e5ba89dfbcc4fd56b3e14716d34806388a8a Mon Sep 17 00:00:00 2001 From: elpolilla Date: Mon, 29 Dec 2008 15:10:21 +0000 Subject: [PATCH] Fixed lots of encoding issues, and improved some adaptors tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40568 --- scrapy/trunk/scrapy/contrib/adaptors/misc.py | 4 +- .../tests/sample_data/adaptors/enc-ascii.html | 17 ++++ .../sample_data/adaptors/enc-cp1252.html | 18 ++++ .../sample_data/adaptors/enc-latin1.html | 17 ++++ .../adaptors/enc-utf8-meta-latin1.html | 18 ++++ .../tests/sample_data/adaptors/enc-utf8.html | 18 ++++ .../sample_data/adaptors/extr_unquoted.xml | 5 + scrapy/trunk/scrapy/tests/test_adaptors.py | 99 ++++++++++++++++--- .../scrapy/tests/test_defaultencoding.py | 9 -- scrapy/trunk/scrapy/utils/iterators.py | 14 +-- scrapy/trunk/scrapy/utils/url.py | 4 +- scrapy/trunk/scrapy/xpath/selector.py | 6 +- 12 files changed, 193 insertions(+), 36 deletions(-) create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html create mode 100644 scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml delete mode 100644 scrapy/trunk/scrapy/tests/test_defaultencoding.py diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py index 02a432fd1..2d064eda7 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -4,7 +4,7 @@ import re from scrapy.xpath.selector import XPathSelector, XPathSelectorList from scrapy.utils.url import canonicalize_url from scrapy.utils.misc import extract_regex -from scrapy.utils.python import flatten +from scrapy.utils.python import flatten, str_to_unicode from scrapy.item.adaptors import adaptize def to_unicode(value): @@ -19,7 +19,7 @@ def to_unicode(value): Output: list of unicodes """ if hasattr(value, '__iter__'): - return [ unicode(v) for v in value ] + return [ str_to_unicode(v) if isinstance(v, basestring) else str_to_unicode(str(v)) for v in value ] else: raise TypeError('to_unicode must receive an iterable.') diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html new file mode 100644 index 000000000..2cd1056bc --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html @@ -0,0 +1,17 @@ + + +Encoding tests + + + + +

+£ +£ +€ +€ +  +

+ + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html new file mode 100644 index 000000000..07ec27edc --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html @@ -0,0 +1,18 @@ + + +Encoding tests + + + + +

+£ +£ +£ +€ +€ +€ +

+ + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html new file mode 100644 index 000000000..76bb432d2 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html @@ -0,0 +1,17 @@ + + +Encoding tests + + + + +

+£ +£ +£ +€ +€ +

+ + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html new file mode 100644 index 000000000..a364718ec --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html @@ -0,0 +1,18 @@ + + +Encoding tests + + + + +

+£ +£ +£ +€ +€ +€ +

+ + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html new file mode 100644 index 000000000..0fce18cb6 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html @@ -0,0 +1,18 @@ + + +Encoding tests + + + + +

+£ +£ +£ +€ +€ +€ +

+ + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml b/scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml new file mode 100644 index 000000000..43c9f6068 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml @@ -0,0 +1,5 @@ + + +test text & &blah&blah +blaheawfds< + diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py index ee26f64f9..6f1a69011 100644 --- a/scrapy/trunk/scrapy/tests/test_adaptors.py +++ b/scrapy/trunk/scrapy/tests/test_adaptors.py @@ -1,22 +1,83 @@ # -*- coding: utf8 -*- +import os import unittest +import re + from scrapy.contrib import adaptors -from scrapy.http import Response, ResponseBody -from scrapy.xpath.selector import XmlXPathSelector +from scrapy.http import Response, ResponseBody, Headers +from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector class AdaptorsTestCase(unittest.TestCase): + def setUp(self): + self.samplesdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample_data', 'adaptors')) + + def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector): + sample_filename = os.path.join(self.samplesdir, sample_filename) + body = file(sample_filename).read() + response = Response(domain=domain, url=url, headers=Headers(headers), status='200', body=body) + return selector(response) + def test_extract(self): - sample_xsel = XmlXPathSelector(text='foobartest') - self.assertEqual(adaptors.extract(sample_xsel.x('/')), - ['foobartest']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')), - ['foobar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), ['foo', 'bar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), ['foo', 'bar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag']) - + def check_extractor(x, pound=True, euro=True): + poundre = re.compile(r'(.*?)') + eurore = re.compile(r'(.*?)') + + if pound: + self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/text()")), [u'\xa3']) + self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/@value")), [u'\xa3']) + self.assertEqual(adaptors.extract(x.re(poundre)), [u'\xa3']) + self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/text()")), [u'\xa3']) + self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/@value")), [u'\xa3']) + self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/text()")), [u'\xa3']) + self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/@value")), [u'\xa3']) + if euro: + self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/text()")), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/@value")), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.re(eurore)), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/text()")), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/@value")), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/text()")), [u'\u20ac']) + self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/@value")), [u'\u20ac']) + + x = self.get_selector('example.com', + 'http://www.example.com/test/utf8', + 'enc-utf8.html', + {}) + check_extractor(x) + + x = self.get_selector('example.com', + 'http://www.example.com/test/latin1', + 'enc-latin1.html', + {}) + check_extractor(x, euro=False) + + x = self.get_selector('example.com', + 'http://www.example.com/test/cp1252', + 'enc-cp1252.html', + {}) + check_extractor(x) + + # HTTP utf-8 | Meta latin1 | Content ascii | using entities + x = self.get_selector('example.com', + 'http://www.example.com/test/ascii', + 'enc-ascii.html', + {'Content-Type': ['text/html; charset=utf-8']}) + check_encoding(x, pound=False, euro=False) + + # Test for inconsistencies between HTTP header encoding and + # META header encoding. It must prefer HTTP header like browsers do + x = self.get_selector('example.com', + 'http://www.example.com/test/utf8-meta-latin1', + 'enc-utf8-meta-latin1.html', + {'Content-Type': ['text/html; charset=utf-8']}) + check_extractor(x) + + + def test_extract_unquoted(self): + x = self.get_selector('example.com', 'http://www.example.com/test_unquoted', 'extr_unquoted.xml', selector=XmlXPathSelector) + self.assertEqual(adaptors.extract_unquoted(x.x('//tag1/text()')), [u'test text & &', u'more test text & >', u'blah&blah']) + self.assertEqual(adaptors.extract_unquoted(x.x('//tag2/text()')), [u'blaheawfds<']) + def test_extract_links(self): test_data = """
@@ -43,44 +104,54 @@ class AdaptorsTestCase(unittest.TestCase): u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/imgs/lala4.jpg']) self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')), [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html']) + def test_to_unicode(self): - self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']), + self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'lulu\xc3\xb1', 1, '\xc3\xa1\xc3\xa9']), [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9']) + def test_regex(self): adaptor = adaptors.Regex(regex=r'href="(.*?)"') self.assertEqual(adaptor(['dsa', 'href="lelelel.net"']), ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) + def test_unquote_all(self): self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) + def test_unquote(self): self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) + def test_remove_tags(self): test_data = ['adsaas
', '
dsadasf
'] self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf']) + def test_remove_root(self): self.assertEqual(adaptors.remove_root(['
lallaadsfsdfdspepepep
']), ['lallaadsfsdfdspepepep
']) + def test_remove_multispaces(self): self.assertEqual(adaptors.clean_spaces([' hello, whats up?', 'testing testingtesting testing']), [' hello, whats up?', 'testing testingtesting testing']) + def test_strip(self): self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), ['hi there, sweety ;D', 'I CAN HAZ TEST??']) self.assertEqual(adaptors.strip(' hello there, this is my test '), 'hello there, this is my test') + def test_drop_empty_elements(self): self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']), [1, 2, 5, 6, 'hi']) + def test_delist(self): self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), diff --git a/scrapy/trunk/scrapy/tests/test_defaultencoding.py b/scrapy/trunk/scrapy/tests/test_defaultencoding.py deleted file mode 100644 index 36c5b782d..000000000 --- a/scrapy/trunk/scrapy/tests/test_defaultencoding.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys -from unittest import TestCase, main - -class DefaultEncodingTest(TestCase): - def test_defaultencoding(self): - self.assertEqual(sys.getdefaultencoding(), 'utf-8') - -if __name__ == "__main__": - main() diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py index d585dcc06..2ff0d5f33 100644 --- a/scrapy/trunk/scrapy/utils/iterators.py +++ b/scrapy/trunk/scrapy/utils/iterators.py @@ -3,16 +3,16 @@ import re, csv from scrapy.xpath import XmlXPathSelector from scrapy.http import Response from scrapy import log -from scrapy.utils.python import re_rsearch +from scrapy.utils.python import re_rsearch, str_to_unicode, unicode_to_str -def _normalize_input(obj): +def _normalize_input(obj, unicode=True): assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): - return obj.body.to_unicode() + return obj.body.to_unicode() if unicode else obj.body.to_string() elif isinstance(obj, str): - return obj.decode('utf-8') + return obj.decode('utf-8') if unicode else obj else: - return obj + return obj if unicode else obj.encode('utf-8') def xmliter(obj, nodename): """Return a iterator of XPathSelector's over all nodes of a XML document, @@ -51,9 +51,9 @@ def csviter(obj, delimiter=None, headers=None): for the returned dictionaries, if not the first row is used. """ def _getrow(csv_r): - return [field.decode() for field in csv_r.next()] + return [str_to_unicode(field) for field in csv_r.next()] - lines = _normalize_input(obj).splitlines(True) + lines = _normalize_input(obj, unicode=False).splitlines(True) if delimiter: csv_r = csv.reader(lines, delimiter=delimiter) else: diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index e16e76c06..8963d90d5 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -9,6 +9,8 @@ import urllib import posixpath import cgi +from scrapy.utils.python import unicode_to_str + def url_is_from_any_domain(url, domains): """Return True if the url belongs to the given domain""" host = urlparse.urlparse(url).hostname @@ -141,7 +143,7 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): For examples see the tests in scrapy.tests.test_utils_url """ - url = url.encode('utf-8') + url = unicode_to_str(url) parts = list(urlparse.urlparse(url)) keyvals = cgi.parse_qsl(parts[4], keep_blank_values) keyvals.sort() diff --git a/scrapy/trunk/scrapy/xpath/selector.py b/scrapy/trunk/scrapy/xpath/selector.py index 7ddd108dc..f614f9f5f 100644 --- a/scrapy/trunk/scrapy/xpath/selector.py +++ b/scrapy/trunk/scrapy/xpath/selector.py @@ -65,13 +65,13 @@ class XPathSelector(object): text = unicode(data, 'utf-8', errors='ignore') if data else u'' elif isinstance(self.xmlNode, libxml2.xmlAttr): # serialization doesn't work sometimes for xmlAttr types - text = unicode(self.xmlNode.content, errors='ignore') + text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') else: data = self.xmlNode.serialize('utf-8') text = unicode(data, 'utf-8', errors='ignore') if data else u'' else: try: - text = unicode(self.xmlNode, errors='ignore') + text = unicode(self.xmlNode, 'utf-8', errors='ignore') except TypeError: # catched when self.xmlNode is a float - see tests text = unicode(self.xmlNode) return text @@ -79,7 +79,7 @@ class XPathSelector(object): def extract_unquoted(self): """Get unescaped contents from the text node (no entities, no CDATA)""" if self.x('self::text()'): - return unicode(self.xmlNode.getContent(), errors='ignore') + return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') else: return u''