From 789cba2bd892727f0494cca70b38eabc3ea7e2b4 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 15 Oct 2009 16:26:05 +0100 Subject: [PATCH] improve csviter memeory usage by use of a generator instead of splitlines --- scrapy/utils/iterators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9f745503f..d528a857d 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,4 +1,5 @@ import re, csv +from cStringIO import StringIO from scrapy.http import Response from scrapy.selector import XmlXPathSelector @@ -48,7 +49,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None): def _getrow(csv_r): return [str_to_unicode(field, encoding) for field in csv_r.next()] - lines = body_or_str(obj, unicode=False).splitlines(True) + lines = StringIO(body_or_str(obj, unicode=False)) if delimiter: csv_r = csv.reader(lines, delimiter=delimiter) else: