RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed encoding-related bug in csviter 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40780
This commit is contained in:
elpolilla 2009-01-27 11:10:42 +00:00
parent 4013497edb
commit b6246cbc37
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/trunk/scrapy/utils/iterators.py
View File

@ -1,5 +1,6 @@
import re, csv
from scrapy.http import Response
from scrapy.xpath import XmlXPathSelector
from scrapy import log
from scrapy.utils.python import re_rsearch, str_to_unicode
@ -28,7 +29,7 @@ def xmliter(obj, nodename):
nodetext = header_start + match.group() + header_end
yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0]
def csviter(obj, delimiter=None, headers=None):
def csviter(obj, delimiter=None, headers=None, encoding=None):
""" Returns an iterator of dictionaries from the given csv object
obj can be:
@ -41,8 +42,9 @@ def csviter(obj, delimiter=None, headers=None):
headers is an iterable that when provided offers the keys
for the returned dictionaries, if not the first row is used.
"""
encoding = obj.encoding if isinstance(obj, Response) else encoding or 'utf-8'
def _getrow(csv_r):
return [str_to_unicode(field) for field in csv_r.next()]
return [str_to_unicode(field, encoding) for field in csv_r.next()]
lines = body_or_str(obj, unicode=False).splitlines(True)
if delimiter: