mirror of https://github.com/scrapy/scrapy.git
Fixed lots of encoding issues, and improved some adaptors tests
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40568
This commit is contained in:
elpolilla
parent
fc3f66bd1c
commit
5eb7e5ba89
|
|
@ -4,7 +4,7 @@ import re
|
|||
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
|
||||
from scrapy.utils.url import canonicalize_url
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
from scrapy.utils.python import flatten, str_to_unicode
|
||||
from scrapy.item.adaptors import adaptize
|
||||
|
||||
def to_unicode(value):
|
||||
|
|
@ -19,7 +19,7 @@ def to_unicode(value):
|
|||
Output: list of unicodes
|
||||
"""
|
||||
if hasattr(value, '__iter__'):
|
||||
return [ unicode(v) for v in value ]
|
||||
return [ str_to_unicode(v) if isinstance(v, basestring) else str_to_unicode(str(v)) for v in value ]
|
||||
else:
|
||||
raise TypeError('to_unicode must receive an iterable.')
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Encoding tests</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>
|
||||
<span class="poundent" value="£">£</span>
|
||||
<span class="poundnum" value="£">£</span>
|
||||
<span class="euroent" value="€">€</span>
|
||||
<span class="euronum" value="€">€</span>
|
||||
<span class="blankspace" value=" "> </span>
|
||||
</p>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Encoding tests</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=windows-1252">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>
|
||||
<span class="pound" value="£">£</span>
|
||||
<span class="poundent" value="£">£</span>
|
||||
<span class="poundnum" value="£">£</span>
|
||||
<span class="euro" value="€">€</span>
|
||||
<span class="euroent" value="€">€</span>
|
||||
<span class="euronum" value="€">€</span>
|
||||
</p>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Encoding tests</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>
|
||||
<span class="pound" value="£">£</span>
|
||||
<span class="poundent" value="£">£</span>
|
||||
<span class="poundnum" value="£">£</span>
|
||||
<span class="euroent" value="€">€</span>
|
||||
<span class="euronum" value="€">€</span>
|
||||
</p>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Encoding tests</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>
|
||||
<span class="pound" value="£">£</span>
|
||||
<span class="poundent" value="£">£</span>
|
||||
<span class="poundnum" value="£">£</span>
|
||||
<span class="euro" value="€">€</span>
|
||||
<span class="euroent" value="€">€</span>
|
||||
<span class="euronum" value="€">€</span>
|
||||
</p>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Encoding tests</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>
|
||||
<span class="pound" value="£">£</span>
|
||||
<span class="poundent" value="£">£</span>
|
||||
<span class="poundnum" value="£">£</span>
|
||||
<span class="euro" value="€">€</span>
|
||||
<span class="euroent" value="€">€</span>
|
||||
<span class="euronum" value="€">€</span>
|
||||
</p>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<?xml version="1.0" ?>
|
||||
<test_parent>
|
||||
<tag1>test text & &<![CDATA[more test text & >]]>blah&blah</tag1>
|
||||
<tag2>blaheawfds<</tag2>
|
||||
</test_parent>
|
||||
|
|
@ -1,22 +1,83 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import os
|
||||
import unittest
|
||||
import re
|
||||
|
||||
from scrapy.contrib import adaptors
|
||||
from scrapy.http import Response, ResponseBody
|
||||
from scrapy.xpath.selector import XmlXPathSelector
|
||||
from scrapy.http import Response, ResponseBody, Headers
|
||||
from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector
|
||||
|
||||
class AdaptorsTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.samplesdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample_data', 'adaptors'))
|
||||
|
||||
def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector):
|
||||
sample_filename = os.path.join(self.samplesdir, sample_filename)
|
||||
body = file(sample_filename).read()
|
||||
response = Response(domain=domain, url=url, headers=Headers(headers), status='200', body=body)
|
||||
return selector(response)
|
||||
|
||||
def test_extract(self):
|
||||
sample_xsel = XmlXPathSelector(text='<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>')
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('/')),
|
||||
['<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')),
|
||||
['<tag1>foo<tag2>bar</tag2></tag1>', '<tag3 value="mytag">test</tag3>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), ['foo', 'bar'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), ['foo', 'bar', 'test'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag'])
|
||||
|
||||
def check_extractor(x, pound=True, euro=True):
|
||||
poundre = re.compile(r'<span class="pound" .*?>(.*?)</span>')
|
||||
eurore = re.compile(r'<span class="euro" .*?>(.*?)</span>')
|
||||
|
||||
if pound:
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/text()")), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/@value")), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.re(poundre)), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/text()")), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/@value")), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/text()")), [u'\xa3'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/@value")), [u'\xa3'])
|
||||
if euro:
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/text()")), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/@value")), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.re(eurore)), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/text()")), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/@value")), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/text()")), [u'\u20ac'])
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/@value")), [u'\u20ac'])
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/utf8',
|
||||
'enc-utf8.html',
|
||||
{})
|
||||
check_extractor(x)
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/latin1',
|
||||
'enc-latin1.html',
|
||||
{})
|
||||
check_extractor(x, euro=False)
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/cp1252',
|
||||
'enc-cp1252.html',
|
||||
{})
|
||||
check_extractor(x)
|
||||
|
||||
# HTTP utf-8 | Meta latin1 | Content ascii | using entities
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/ascii',
|
||||
'enc-ascii.html',
|
||||
{'Content-Type': ['text/html; charset=utf-8']})
|
||||
check_encoding(x, pound=False, euro=False)
|
||||
|
||||
# Test for inconsistencies between HTTP header encoding and
|
||||
# META header encoding. It must prefer HTTP header like browsers do
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/utf8-meta-latin1',
|
||||
'enc-utf8-meta-latin1.html',
|
||||
{'Content-Type': ['text/html; charset=utf-8']})
|
||||
check_extractor(x)
|
||||
|
||||
|
||||
def test_extract_unquoted(self):
|
||||
x = self.get_selector('example.com', 'http://www.example.com/test_unquoted', 'extr_unquoted.xml', selector=XmlXPathSelector)
|
||||
self.assertEqual(adaptors.extract_unquoted(x.x('//tag1/text()')), [u'test text & &', u'more test text & >', u'blah&blah'])
|
||||
self.assertEqual(adaptors.extract_unquoted(x.x('//tag2/text()')), [u'blaheawfds<'])
|
||||
|
||||
def test_extract_links(self):
|
||||
test_data = """<html><body>
|
||||
<div>
|
||||
|
|
@ -43,44 +104,54 @@ class AdaptorsTestCase(unittest.TestCase):
|
|||
u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/imgs/lala4.jpg'])
|
||||
self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')),
|
||||
[u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
|
||||
|
||||
|
||||
def test_to_unicode(self):
|
||||
self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
|
||||
self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'lulu\xc3\xb1', 1, '\xc3\xa1\xc3\xa9']),
|
||||
[u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
|
||||
|
||||
|
||||
def test_regex(self):
|
||||
adaptor = adaptors.Regex(regex=r'href="(.*?)"')
|
||||
self.assertEqual(adaptor(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
|
||||
'<a href="das.biz">href="lelelel.net"</a>']),
|
||||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
|
||||
|
||||
def test_unquote_all(self):
|
||||
self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
|
||||
def test_unquote(self):
|
||||
self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
|
||||
def test_remove_tags(self):
|
||||
test_data = ['<a href="lala">adsaas<br /></a>', '<div id="1"><table>dsadasf</table></div>']
|
||||
self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
|
||||
|
||||
|
||||
def test_remove_root(self):
|
||||
self.assertEqual(adaptors.remove_root(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
|
||||
['lallaa<a href="coso">dsfsdfds</a>pepepep<br />'])
|
||||
|
||||
|
||||
def test_remove_multispaces(self):
|
||||
self.assertEqual(adaptors.clean_spaces([' hello, whats up?', 'testing testingtesting testing']),
|
||||
[' hello, whats up?', 'testing testingtesting testing'])
|
||||
|
||||
|
||||
def test_strip(self):
|
||||
self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
|
||||
['hi there, sweety ;D', 'I CAN HAZ TEST??'])
|
||||
self.assertEqual(adaptors.strip(' hello there, this is my test '),
|
||||
'hello there, this is my test')
|
||||
|
||||
|
||||
def test_drop_empty_elements(self):
|
||||
self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']),
|
||||
[1, 2, 5, 6, 'hi'])
|
||||
|
||||
|
||||
def test_delist(self):
|
||||
self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import sys
|
||||
from unittest import TestCase, main
|
||||
|
||||
class DefaultEncodingTest(TestCase):
|
||||
def test_defaultencoding(self):
|
||||
self.assertEqual(sys.getdefaultencoding(), 'utf-8')
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -3,16 +3,16 @@ import re, csv
|
|||
from scrapy.xpath import XmlXPathSelector
|
||||
from scrapy.http import Response
|
||||
from scrapy import log
|
||||
from scrapy.utils.python import re_rsearch
|
||||
from scrapy.utils.python import re_rsearch, str_to_unicode, unicode_to_str
|
||||
|
||||
def _normalize_input(obj):
|
||||
def _normalize_input(obj, unicode=True):
|
||||
assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
|
||||
if isinstance(obj, Response):
|
||||
return obj.body.to_unicode()
|
||||
return obj.body.to_unicode() if unicode else obj.body.to_string()
|
||||
elif isinstance(obj, str):
|
||||
return obj.decode('utf-8')
|
||||
return obj.decode('utf-8') if unicode else obj
|
||||
else:
|
||||
return obj
|
||||
return obj if unicode else obj.encode('utf-8')
|
||||
|
||||
def xmliter(obj, nodename):
|
||||
"""Return a iterator of XPathSelector's over all nodes of a XML document,
|
||||
|
|
@ -51,9 +51,9 @@ def csviter(obj, delimiter=None, headers=None):
|
|||
for the returned dictionaries, if not the first row is used.
|
||||
"""
|
||||
def _getrow(csv_r):
|
||||
return [field.decode() for field in csv_r.next()]
|
||||
return [str_to_unicode(field) for field in csv_r.next()]
|
||||
|
||||
lines = _normalize_input(obj).splitlines(True)
|
||||
lines = _normalize_input(obj, unicode=False).splitlines(True)
|
||||
if delimiter:
|
||||
csv_r = csv.reader(lines, delimiter=delimiter)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import urllib
|
|||
import posixpath
|
||||
import cgi
|
||||
|
||||
from scrapy.utils.python import unicode_to_str
|
||||
|
||||
def url_is_from_any_domain(url, domains):
|
||||
"""Return True if the url belongs to the given domain"""
|
||||
host = urlparse.urlparse(url).hostname
|
||||
|
|
@ -141,7 +143,7 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False):
|
|||
For examples see the tests in scrapy.tests.test_utils_url
|
||||
"""
|
||||
|
||||
url = url.encode('utf-8')
|
||||
url = unicode_to_str(url)
|
||||
parts = list(urlparse.urlparse(url))
|
||||
keyvals = cgi.parse_qsl(parts[4], keep_blank_values)
|
||||
keyvals.sort()
|
||||
|
|
|
|||
|
|
@ -65,13 +65,13 @@ class XPathSelector(object):
|
|||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
elif isinstance(self.xmlNode, libxml2.xmlAttr):
|
||||
# serialization doesn't work sometimes for xmlAttr types
|
||||
text = unicode(self.xmlNode.content, errors='ignore')
|
||||
text = unicode(self.xmlNode.content, 'utf-8', errors='ignore')
|
||||
else:
|
||||
data = self.xmlNode.serialize('utf-8')
|
||||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
else:
|
||||
try:
|
||||
text = unicode(self.xmlNode, errors='ignore')
|
||||
text = unicode(self.xmlNode, 'utf-8', errors='ignore')
|
||||
except TypeError: # catched when self.xmlNode is a float - see tests
|
||||
text = unicode(self.xmlNode)
|
||||
return text
|
||||
|
|
@ -79,7 +79,7 @@ class XPathSelector(object):
|
|||
def extract_unquoted(self):
|
||||
"""Get unescaped contents from the text node (no entities, no CDATA)"""
|
||||
if self.x('self::text()'):
|
||||
return unicode(self.xmlNode.getContent(), errors='ignore')
|
||||
return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
|
||||
else:
|
||||
return u''
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue