improve csviter memeory usage by use of a generator instead of splitlines

This commit is contained in:
daniel 2009-10-15 16:26:05 +01:00
parent d6735b3952
commit 789cba2bd8
1 changed files with 2 additions and 1 deletions

View File

@ -1,4 +1,5 @@
import re, csv
from cStringIO import StringIO
from scrapy.http import Response
from scrapy.selector import XmlXPathSelector
@ -48,7 +49,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
def _getrow(csv_r):
return [str_to_unicode(field, encoding) for field in csv_r.next()]
lines = body_or_str(obj, unicode=False).splitlines(True)
lines = StringIO(body_or_str(obj, unicode=False))
if delimiter:
csv_r = csv.reader(lines, delimiter=delimiter)
else: