From 5fddea58a75cacbecf0e7d340f08767e2591005c Mon Sep 17 00:00:00 2001 From: samus_ Date: Fri, 17 Oct 2008 21:53:40 +0000 Subject: [PATCH] created csv iterator to replace CSVParser iteration facilities --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40319 --- scrapy/trunk/scrapy/utils/iterators.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 scrapy/trunk/scrapy/utils/iterators.py diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py new file mode 100644 index 000000000..5f989d0a3 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/iterators.py @@ -0,0 +1,23 @@ +import csv + +from scrapy.http import Response +from scrapy import log + +def csv_iter(response, delimiter=None, headers=None): + if delimiter: + csv_r = csv.reader(response.body.to_unicode().split('\n'), delimiter=delimiter) + else: + csv_r = csv.reader(response.body.to_unicode().split('\n')) + + if not headers: + headers = csv_r.next() + + while True: + node = csv_r.next() + + if len(node) != len(headers): + log.msg("ignoring node %d (length: %d, should be: %d)" % (csv_r.line_num, len(node), len(headers)), log.WARNING) + continue + + yield dict(zip(headers, node)) +