mirror of https://github.com/scrapy/scrapy.git
removed unused items_to_csv function
This commit is contained in:
parent
6dfda357ef
commit
50de128e0d
|
|
@ -1,55 +1,11 @@
|
|||
import unittest
|
||||
from cStringIO import StringIO
|
||||
|
||||
from scrapy.utils.misc import items_to_csv, load_object, arg_to_iter
|
||||
from scrapy.utils.misc import load_object, arg_to_iter
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
class UtilsMiscTestCase(unittest.TestCase):
|
||||
|
||||
def test_items_to_csv(self):
|
||||
item_1 = ScrapedItem()
|
||||
item_1._hidden = '543565'
|
||||
item_1.id = '3213'
|
||||
item_1.name = 'Item 1'
|
||||
item_1.description = 'Really cute item'
|
||||
item_1.url = 'http://dummyurl.com'
|
||||
|
||||
item_2 = ScrapedItem()
|
||||
item_2._hidden = 'lala'
|
||||
item_2.id = '1234'
|
||||
item_2.name = 'Item 2'
|
||||
item_2.description = 'This item rocks!'
|
||||
item_2.url = 'http://dummyurl.com/2'
|
||||
item_2.supplier = 'A random supplier'
|
||||
|
||||
file = StringIO()
|
||||
items_to_csv(file, [item_1, item_2])
|
||||
file.reset()
|
||||
self.assertEqual(file.read(),
|
||||
'"description";"id";"name";"url"\r\n' +
|
||||
'"Really cute item";"3213";"Item 1";"http://dummyurl.com"\r\n' +
|
||||
'"This item rocks!";"1234";"Item 2";"http://dummyurl.com/2"\r\n')
|
||||
|
||||
file = StringIO()
|
||||
items_to_csv(file, [item_2, item_1])
|
||||
file.reset()
|
||||
self.assertEqual(file.read(),
|
||||
'"description";"id";"name";"supplier";"url"\r\n' +
|
||||
'"This item rocks!";"1234";"Item 2";"A random supplier";"http://dummyurl.com/2"\r\n' +
|
||||
'"Really cute item";"3213";"Item 1";"";"http://dummyurl.com"\r\n')
|
||||
|
||||
file = StringIO()
|
||||
items_to_csv(file, [item_1, item_2], headers=['id', 'name'], delimiter=',')
|
||||
file.reset()
|
||||
self.assertEqual(file.read(),
|
||||
'"id","name"\r\n' +
|
||||
'"3213","Item 1"\r\n' +
|
||||
'"1234","Item 2"\r\n')
|
||||
|
||||
file = StringIO()
|
||||
items_to_csv(file, [])
|
||||
self.assertEqual(file.tell(), 0)
|
||||
|
||||
def test_load_object(self):
|
||||
obj = load_object('scrapy.utils.misc.load_object')
|
||||
assert obj is load_object
|
||||
|
|
|
|||
|
|
@ -94,33 +94,6 @@ def extract_regex(regex, text, encoding):
|
|||
else:
|
||||
return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings]
|
||||
|
||||
def items_to_csv(file, items, delimiter=';', headers=None):
|
||||
"""
|
||||
This function takes a list of items and stores their attributes
|
||||
in a csv file given in 'file' (which can be either a descriptor, or a filename).
|
||||
The saved attributes are either the ones found in the 'headers' parameter
|
||||
(if specified) or the first item's list of public attributes.
|
||||
The written file will be encoded as utf-8.
|
||||
"""
|
||||
if not items or not hasattr(items, '__iter__'):
|
||||
return
|
||||
|
||||
if isinstance(file, basestring):
|
||||
file = open(file, 'ab+')
|
||||
csv_file = csv.writer(file, delimiter=delimiter, quoting=csv.QUOTE_ALL)
|
||||
header = headers or sorted([key for key in items[0].__dict__.keys() if not key.startswith('_')])
|
||||
if not file.tell():
|
||||
csv_file.writerow(header)
|
||||
|
||||
for item in items:
|
||||
row = []
|
||||
for attrib in header:
|
||||
value = getattr(item, attrib, None)
|
||||
value = unicode_to_str(value) if isinstance(value, basestring) else value
|
||||
row.append(value)
|
||||
csv_file.writerow(row)
|
||||
|
||||
|
||||
def md5sum(buffer):
|
||||
"""Calculate the md5 checksum of a file
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue