- Improved unquote_markup by using generators instead of lists

- Added possibility of specifying headers in items_to_csv

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40552

scrapy/trunk/scrapy/tests/test_utils_misc.py

@@ -44,6 +44,14 @@ class UtilsMiscTestCase(unittest.TestCase):
             '"This item rocks!";"1234";"Item 2";"A random supplier";"http://dummyurl.com/2"\r\n' +
             '"Really cute item";"3213";"Item 1";"";"http://dummyurl.com"\r\n')
 
+        file = StringIO()
+        items_to_csv(file, [item_1, item_2], headers=['id', 'name'], delimiter=',')
+        file.reset()
+        self.assertEqual(file.read(),
+            '"id","name"\r\n' +
+            '"3213","Item 1"\r\n' +
+            '"1234","Item 2"\r\n')
+
         file = StringIO()
         items_to_csv(file, [])
         self.assertEqual(file.tell(), 0)

scrapy/trunk/scrapy/utils/markup.py

@@ -110,15 +110,13 @@ def unquote_markup(text, keep=(), remove_illegal=True):
     _cdata_re = re.compile(r'((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))', re.DOTALL)
 
     def _get_fragments(txt, pattern):
-        fragments = []
         offset = 0
         for match in pattern.finditer(txt):
             match_s, match_e = match.span(1)
-            fragments.append(txt[offset:match_s])
-            fragments.append(match)
+            yield txt[offset:match_s]
+            yield match
             offset = match_e
-        fragments.append(txt[offset:])
-        return fragments
+        yield txt[offset:]
 
     text = str_to_unicode(text)
     ret_text = u''

scrapy/trunk/scrapy/utils/misc.py

@@ -128,12 +128,12 @@ def render_templatefile(path, **kwargs):
         file.write(content)
 
 
-def items_to_csv(file, items, delimiter=';'):
+def items_to_csv(file, items, delimiter=';', headers=None):
     """
     This function takes a list of items and stores their attributes
     in a csv file given in 'file' (which can be either a descriptor, or a filename).
-    The attributes are the ones found in the first item of the list, so
-    if it lacks any attribute that other item has, that attribute will be missing.
+    The saved attributes are either the ones found in the 'headers' parameter
+    (if specified) or the first item's list of public attributes.
     The written file will be encoded as utf-8.
     """
     if not items or not hasattr(items, '__iter__'):
@@ -142,7 +142,7 @@ def items_to_csv(file, items, delimiter=';'):
     if isinstance(file, basestring):
         file = open(file, 'a+')
     csv_file = csv.writer(file, delimiter=delimiter, quoting=csv.QUOTE_ALL)
-    header = sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')])
+    header = headers or sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')])
     if not file.tell():
         csv_file.writerow(header)