diff --git a/scrapy/trunk/scrapy/tests/test_utils_misc.py b/scrapy/trunk/scrapy/tests/test_utils_misc.py index b1dacaa93..1f6430d9b 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_misc.py +++ b/scrapy/trunk/scrapy/tests/test_utils_misc.py @@ -44,6 +44,14 @@ class UtilsMiscTestCase(unittest.TestCase): '"This item rocks!";"1234";"Item 2";"A random supplier";"http://dummyurl.com/2"\r\n' + '"Really cute item";"3213";"Item 1";"";"http://dummyurl.com"\r\n') + file = StringIO() + items_to_csv(file, [item_1, item_2], headers=['id', 'name'], delimiter=',') + file.reset() + self.assertEqual(file.read(), + '"id","name"\r\n' + + '"3213","Item 1"\r\n' + + '"1234","Item 2"\r\n') + file = StringIO() items_to_csv(file, []) self.assertEqual(file.tell(), 0) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index f3e585426..da17ea84c 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -110,15 +110,13 @@ def unquote_markup(text, keep=(), remove_illegal=True): _cdata_re = re.compile(r'((?P.*?)(?P\]\]>))', re.DOTALL) def _get_fragments(txt, pattern): - fragments = [] offset = 0 for match in pattern.finditer(txt): match_s, match_e = match.span(1) - fragments.append(txt[offset:match_s]) - fragments.append(match) + yield txt[offset:match_s] + yield match offset = match_e - fragments.append(txt[offset:]) - return fragments + yield txt[offset:] text = str_to_unicode(text) ret_text = u'' diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index 708cc8c90..ea926cdde 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -128,12 +128,12 @@ def render_templatefile(path, **kwargs): file.write(content) -def items_to_csv(file, items, delimiter=';'): +def items_to_csv(file, items, delimiter=';', headers=None): """ This function takes a list of items and stores their attributes in a csv file given in 'file' (which can be either a descriptor, or a filename). - The attributes are the ones found in the first item of the list, so - if it lacks any attribute that other item has, that attribute will be missing. + The saved attributes are either the ones found in the 'headers' parameter + (if specified) or the first item's list of public attributes. The written file will be encoded as utf-8. """ if not items or not hasattr(items, '__iter__'): @@ -142,7 +142,7 @@ def items_to_csv(file, items, delimiter=';'): if isinstance(file, basestring): file = open(file, 'a+') csv_file = csv.writer(file, delimiter=delimiter, quoting=csv.QUOTE_ALL) - header = sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')]) + header = headers or sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')]) if not file.tell(): csv_file.writerow(header)