diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py new file mode 100644 index 000000000..5f989d0a3 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/iterators.py @@ -0,0 +1,23 @@ +import csv + +from scrapy.http import Response +from scrapy import log + +def csv_iter(response, delimiter=None, headers=None): + if delimiter: + csv_r = csv.reader(response.body.to_unicode().split('\n'), delimiter=delimiter) + else: + csv_r = csv.reader(response.body.to_unicode().split('\n')) + + if not headers: + headers = csv_r.next() + + while True: + node = csv_r.next() + + if len(node) != len(headers): + log.msg("ignoring node %d (length: %d, should be: %d)" % (csv_r.line_num, len(node), len(headers)), log.WARNING) + continue + + yield dict(zip(headers, node)) +