From f59f1c8bc07dc378e7898ee94079d1def964d2fd Mon Sep 17 00:00:00 2001
From: elpolilla <none@none>
Date: Fri, 26 Dec 2008 16:10:03 +0000
Subject: [PATCH] Added items_to_csv function

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40551
---
 scrapy/trunk/scrapy/tests/test_utils_misc.py | 40 +++++++++++++++++++-
 scrapy/trunk/scrapy/utils/misc.py            | 28 +++++++++++++-
 2 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/scrapy/trunk/scrapy/tests/test_utils_misc.py b/scrapy/trunk/scrapy/tests/test_utils_misc.py
index 994b165d5..b1dacaa93 100644
--- a/scrapy/trunk/scrapy/tests/test_utils_misc.py
+++ b/scrapy/trunk/scrapy/tests/test_utils_misc.py
@@ -1,7 +1,9 @@
 import unittest
+from cStringIO import StringIO
 
-from scrapy.utils.misc import hash_values
+from scrapy.utils.misc import hash_values, items_to_csv
 from scrapy.core.exceptions import UsageError
+from scrapy.item import ScrapedItem
 
 class UtilsMiscTestCase(unittest.TestCase):
     def test_hash_values(self):
@@ -10,5 +12,41 @@ class UtilsMiscTestCase(unittest.TestCase):
 
         self.assertRaises(UsageError, hash_values, 'some', None, 'value')
 
+    def test_items_to_csv(self):
+        item_1 = ScrapedItem()
+        item_1.attribute('_hidden', '543565')
+        item_1.attribute('id', '3213')
+        item_1.attribute('name', 'Item 1')
+        item_1.attribute('description', 'Really cute item')
+        item_1.attribute('url', 'http://dummyurl.com')
+
+        item_2 = ScrapedItem()
+        item_2.attribute('_hidden', 'lala')
+        item_2.attribute('id', '1234')
+        item_2.attribute('name', 'Item 2')
+        item_2.attribute('description', 'This item rocks!')
+        item_2.attribute('url', 'http://dummyurl.com/2')
+        item_2.attribute('supplier', 'A random supplier')
+
+        file = StringIO()
+        items_to_csv(file, [item_1, item_2])
+        file.reset()
+        self.assertEqual(file.read(),
+            '"description";"id";"name";"url"\r\n' +
+            '"Really cute item";"3213";"Item 1";"http://dummyurl.com"\r\n' +
+            '"This item rocks!";"1234";"Item 2";"http://dummyurl.com/2"\r\n')
+
+        file = StringIO()
+        items_to_csv(file, [item_2, item_1])
+        file.reset()
+        self.assertEqual(file.read(),
+            '"description";"id";"name";"supplier";"url"\r\n' +
+            '"This item rocks!";"1234";"Item 2";"A random supplier";"http://dummyurl.com/2"\r\n' +
+            '"Really cute item";"3213";"Item 1";"";"http://dummyurl.com"\r\n')
+
+        file = StringIO()
+        items_to_csv(file, [])
+        self.assertEqual(file.tell(), 0)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py
index 39ddee5a7..708cc8c90 100644
--- a/scrapy/trunk/scrapy/utils/misc.py
+++ b/scrapy/trunk/scrapy/utils/misc.py
@@ -6,11 +6,12 @@ from __future__ import with_statement
 import re
 import string
 import hashlib
+import csv
 
 from twisted.internet import defer
 
 from scrapy.core.exceptions import UsageError
-from scrapy.utils.python import flatten
+from scrapy.utils.python import flatten, unicode_to_str
 from scrapy.utils.markup import remove_entities
 from scrapy.utils.defer import defer_succeed
 
@@ -127,3 +128,28 @@ def render_templatefile(path, **kwargs):
         file.write(content)
 
 
+def items_to_csv(file, items, delimiter=';'):
+    """
+    This function takes a list of items and stores their attributes
+    in a csv file given in 'file' (which can be either a descriptor, or a filename).
+    The attributes are the ones found in the first item of the list, so
+    if it lacks any attribute that other item has, that attribute will be missing.
+    The written file will be encoded as utf-8.
+    """
+    if not items or not hasattr(items, '__iter__'):
+        return
+
+    if isinstance(file, basestring):
+        file = open(file, 'a+')
+    csv_file = csv.writer(file, delimiter=delimiter, quoting=csv.QUOTE_ALL)
+    header = sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')])
+    if not file.tell():
+        csv_file.writerow(header)
+
+    for item in items:
+        row = []
+        for attrib in header:
+            value = getattr(item, attrib, None)
+            value = unicode_to_str(value) if isinstance(value, basestring) else value
+            row.append(value)
+        csv_file.writerow(row)