mirror of https://github.com/scrapy/scrapy.git
added new item exporter tests, introduced some api changes
This commit is contained in:
Ismael Carnales
parent
907fad6da8
commit
314e8dea43
|
|
@ -20,8 +20,11 @@ Using Item Exporters
|
|||
In order to use a Item Exporter, you must instantiate it with its required
|
||||
args. Different exporters require different args, so check each exporter
|
||||
documentation to be sure, in :ref:`topics-exporters-reference`. After you have
|
||||
instantiated you exporter, you have to call the
|
||||
:meth:`~BaseItemExporter.export` method with each item you want to export.
|
||||
instantiated you exporter, you call the method
|
||||
:meth:`~BaseItemExporter.start_exporting` in order to initialize the exporting
|
||||
proces, then you call :meth:`~BaseItemExporter.export_item` method for each
|
||||
item you want to export, and finally call
|
||||
:meth:`~BaseItemExporter.finish_exporting` to finalize the exporting process.
|
||||
|
||||
Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline
|
||||
<topics-item-pipeline>`::
|
||||
|
|
@ -38,13 +41,14 @@ Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline
|
|||
def domain_open(self, domain):
|
||||
self.file = open('%s_products.xml' % domain)
|
||||
self.exporter = XmlItemExporter(self.file)
|
||||
self.exporter.start_exporting()
|
||||
|
||||
def domain_closed(self, domain):
|
||||
self.exporter.close()
|
||||
self.exporter.finish_exporting()
|
||||
self.file.close()
|
||||
|
||||
def process_item(self, domain, item):
|
||||
self.exporter.export(item)
|
||||
self.exporter.export_item(item)
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -120,7 +124,7 @@ BaseItemExporter
|
|||
|
||||
This is the base class for all Item Exporters, and it's an abstract class.
|
||||
|
||||
.. method:: export(item)
|
||||
.. method:: export_item(item)
|
||||
|
||||
Exports the item to the specific exporter format. This method must be
|
||||
implemented in subclasses.
|
||||
|
|
@ -130,10 +134,16 @@ BaseItemExporter
|
|||
Serializes the field value to ``str``. You can override this method in
|
||||
custom Item Exporters.
|
||||
|
||||
.. method:: close()
|
||||
.. method:: start_exporting()
|
||||
|
||||
Called when there are no more items to export, so the exporter can close
|
||||
the serialization, for those formats that require it (like XML).
|
||||
Makes the exporter initialize the export process, in here exporters may
|
||||
output information required by the exporter's format.
|
||||
|
||||
.. method:: finish_exporting()
|
||||
|
||||
You must call it when there are no more items to export, so the exporter
|
||||
can close the serialization output, for those formats that require it
|
||||
(like XML).
|
||||
|
||||
.. attribute:: fields_to_export
|
||||
|
||||
|
|
@ -196,9 +206,11 @@ CsvItemExporter
|
|||
|
||||
.. attribute:: include_headers_line
|
||||
|
||||
If ``True`` the first line in the CSV export will include the name of the
|
||||
fields columns, taken from the :attr:`BaseItemExporter.fields_to_export`
|
||||
attribute. Defaults to ``False``.
|
||||
Makes the exporter output a header line with the field names taken from
|
||||
:attr:`BaseItemExporter.fields_to_export` so that attribute must also be
|
||||
set in order to work.
|
||||
|
||||
Defaults to ``False``.
|
||||
|
||||
.. _csv.writer: http://docs.python.org/library/csv.html#csv.writer
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class BaseItemExporter(object):
|
|||
fields_to_export = None
|
||||
export_empty_fields = False
|
||||
|
||||
def export(self, item):
|
||||
def export_item(self, item):
|
||||
raise NotImplementedError
|
||||
|
||||
def serialize(self, field, name, value):
|
||||
|
|
@ -43,9 +43,13 @@ class BaseItemExporter(object):
|
|||
nonempty_fields)
|
||||
return [(k, item.get(k, default_value)) for k in field_iter]
|
||||
|
||||
def close(self):
|
||||
def start_exporting(self):
|
||||
pass
|
||||
|
||||
def finish_exporting(self):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class XmlItemExporter(BaseItemExporter):
|
||||
|
||||
|
|
@ -55,15 +59,18 @@ class XmlItemExporter(BaseItemExporter):
|
|||
def __init__(self, file):
|
||||
super(XmlItemExporter, self).__init__()
|
||||
self.xg = XMLGenerator(file)
|
||||
|
||||
def start_exporting(self):
|
||||
self.xg.startDocument()
|
||||
self.xg.startElement(self.root_element, {})
|
||||
|
||||
def export(self, item):
|
||||
def export_item(self, item):
|
||||
self.xg.startElement(self.item_element, {})
|
||||
for field, value in self._get_fields_to_export(item, default_value=''):
|
||||
self._export_xml_field(item.fields[field], field, value)
|
||||
self.xg.endElement(self.item_element)
|
||||
|
||||
def close(self):
|
||||
def finish_exporting(self):
|
||||
self.xg.endElement(self.root_element)
|
||||
self.xg.endDocument()
|
||||
|
||||
|
|
@ -81,15 +88,18 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(CsvItemExporter, self).__init__()
|
||||
self.csv_writer = csv.writer(*args, **kwargs)
|
||||
|
||||
def start_exporting(self):
|
||||
if self.include_headers_line:
|
||||
if not self.fields_to_export:
|
||||
raise RuntimeError("To use include_headers_line you must " \
|
||||
"define fields_to_export attribute")
|
||||
raise RuntimeError("You must set fields_to_export in order to" + \
|
||||
" use include_headers_line")
|
||||
self.csv_writer.writerow(self.fields_to_export)
|
||||
|
||||
def export(self, item):
|
||||
def export_item(self, item):
|
||||
fields = self._get_fields_to_export(item, default_value='', \
|
||||
include_empty=True)
|
||||
|
||||
values = [x[1] for x in fields]
|
||||
self.csv_writer.writerow(values)
|
||||
|
||||
|
|
@ -100,7 +110,7 @@ class PickleItemExporter(BaseItemExporter):
|
|||
super(PickleItemExporter, self).__init__()
|
||||
self.pickler = Pickler(*args, **kwargs)
|
||||
|
||||
def export(self, item):
|
||||
def export_item(self, item):
|
||||
self.pickler.dump(dict(self._get_fields_to_export(item)))
|
||||
|
||||
|
||||
|
|
@ -110,6 +120,6 @@ class PprintItemExporter(BaseItemExporter):
|
|||
super(PprintItemExporter, self).__init__()
|
||||
self.file = file
|
||||
|
||||
def export(self, item):
|
||||
def export_item(self, item):
|
||||
itemdict = dict(self._get_fields_to_export(item))
|
||||
self.file.write(pprint.pformat(itemdict) + '\n')
|
||||
|
|
|
|||
|
|
@ -14,4 +14,4 @@ class JsonLinesItemExporter(BaseItemExporter):
|
|||
|
||||
def export(self, item):
|
||||
itemdict = dict(self._get_fields_to_export(item))
|
||||
self.file.write(self.encoder(itemdict) + '\n')
|
||||
self.file.write(self.encoder.encode(itemdict) + '\n')
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ from twisted.trial import unittest
|
|||
|
||||
from scrapy.newitem import Item, Field
|
||||
|
||||
# FIXME: fix tests
|
||||
"""
|
||||
from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
|
||||
PickleItemExporter, CsvItemExporter, XmlItemExporter
|
||||
|
||||
|
|
@ -15,108 +13,128 @@ class TestItem(Item):
|
|||
age = Field()
|
||||
|
||||
|
||||
class BaseItemExporterTest(unittest.TestCase):
|
||||
|
||||
class BaseTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.i = TestItem(name=u'John', age='22')
|
||||
self.ie = BaseItemExporter()
|
||||
|
||||
self.output = StringIO()
|
||||
|
||||
class BaseItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
self.assertRaises(NotImplementedError, self.ie.export_item, self.i)
|
||||
|
||||
ie = BaseItemExporter()
|
||||
|
||||
self.assertRaises(NotImplementedError, ie.export, i)
|
||||
|
||||
def test_default_serializer(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
|
||||
ie = BaseItemExporter()
|
||||
|
||||
self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual( ie._serialize_field(i.fields['age'], 'age', i['age']), '22')
|
||||
def test_serialize(self):
|
||||
self.assertEqual(self.ie.serialize( \
|
||||
self.i.fields['name'], 'name', self.i['name']), 'John')
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(self.i.fields['age'], 'age', self.i['age']), '22')
|
||||
|
||||
def test_exporter_custom_serializer(self):
|
||||
class CustomItemExporter(BaseItemExporter):
|
||||
def serialize_age(self, field, name, value):
|
||||
return str(value + 1)
|
||||
|
||||
i = TestItem(name=u'John', age=22)
|
||||
def serialize(self, field, name, value):
|
||||
if name == 'age':
|
||||
return str(int(value) + 1)
|
||||
else:
|
||||
return super(CustomItemExporter, self).serialize(field, \
|
||||
name, value)
|
||||
|
||||
ie = CustomItemExporter()
|
||||
|
||||
self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '23')
|
||||
self.assertEqual( \
|
||||
ie.serialize(self.i.fields['name'], 'name', self.i['name']), 'John')
|
||||
self.assertEqual(
|
||||
ie.serialize(self.i.fields['age'], 'age', self.i['age']), '23')
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
class CustomField(Field):
|
||||
def serializer(self, field, name, value):
|
||||
return str(value + 2)
|
||||
def custom_serializer(value):
|
||||
return str(int(value) + 2)
|
||||
|
||||
class CustomFieldItem(Item):
|
||||
name = Field()
|
||||
age = CustomField()
|
||||
age = Field(serializer=custom_serializer)
|
||||
|
||||
i = CustomFieldItem(name=u'John', age=22)
|
||||
i = CustomFieldItem(name=u'John', age='22')
|
||||
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(i.fields['age'], 'age', i['age']), '24')
|
||||
|
||||
def test_fields_to_export(self):
|
||||
ie = BaseItemExporter()
|
||||
ie.fields_to_export = ['name']
|
||||
|
||||
self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '24')
|
||||
self.assertEqual(ie._get_fields_to_export(self.i), [('name', 'John')])
|
||||
|
||||
|
||||
class PprintItemExporterTest(unittest.TestCase):
|
||||
class PprintItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
ie = PprintItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
output = StringIO()
|
||||
ie = PprintItemExporter(output)
|
||||
ie.export(i)
|
||||
|
||||
self.assertEqual(output.getvalue(), "{'age': 22, 'name': u'John'}\n")
|
||||
self.assertEqual(self.output.getvalue(), "{'age': '22', 'name': u'John'}\n")
|
||||
|
||||
|
||||
class PickleItemExporterTest(unittest.TestCase):
|
||||
class PickleItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
|
||||
output = StringIO()
|
||||
ie = PickleItemExporter(output)
|
||||
ie.export(i)
|
||||
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
poutput = StringIO()
|
||||
p = Pickler(poutput)
|
||||
p.dump(dict(i))
|
||||
p.dump(dict(self.i))
|
||||
|
||||
self.assertEqual(output.getvalue(), poutput.getvalue())
|
||||
|
||||
|
||||
class CsvItemExporterTest(unittest.TestCase):
|
||||
class CsvItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
ie = CsvItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
output = StringIO()
|
||||
ie = CsvItemExporter(output)
|
||||
ie.fields_to_export = i.fields.keys()
|
||||
ie.export(i)
|
||||
self.assertEqual(self.output.getvalue(), '22,John\r\n')
|
||||
|
||||
self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n')
|
||||
def test_header(self):
|
||||
ie = CsvItemExporter(self.output)
|
||||
ie.include_headers_line = True
|
||||
|
||||
self.assertRaises(RuntimeError, ie.start_exporting)
|
||||
|
||||
ie.fields_to_export = self.i.fields.keys()
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\r\n')
|
||||
|
||||
|
||||
class XmlItemExporterTest(unittest.TestCase):
|
||||
class XmlItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
i = TestItem(name=u'John', age=22)
|
||||
ie = XmlItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
output = StringIO()
|
||||
ie = XmlItemExporter(output)
|
||||
ie.fields_to_export = i.fields.keys()
|
||||
ie.export(i)
|
||||
expected_value = '<?xml version="1.0" encoding="iso-8859-1"?>\n<items><item><age>22</age><name>John</name></item></items>'
|
||||
|
||||
self.assertEqual(output.getvalue(), \
|
||||
'<?xml version="1.0" encoding="iso-8859-1"?>\n<items><item><age>22</age><name>John</name></item>')
|
||||
self.assertEqual(self.output.getvalue(), expected_value)
|
||||
|
||||
|
||||
class JSONItemExporterTest(unittest.TestCase):
|
||||
class JSONItemExporterTest(BaseTest):
|
||||
|
||||
def setUp(self):
|
||||
try:
|
||||
|
|
@ -127,17 +145,19 @@ class JSONItemExporterTest(unittest.TestCase):
|
|||
except ImportError:
|
||||
raise unittest.SkipTest("simplejson module not available")
|
||||
|
||||
from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter
|
||||
|
||||
super(JSONItemExporterTest, self).__init__()
|
||||
|
||||
def test_export(self):
|
||||
from scrapy.contrib.exporter.jsonexporter import JSONItemExporter
|
||||
|
||||
output = StringIO()
|
||||
ie = JSONItemExporter(output)
|
||||
i = TestItem(name=u'John', age=22)
|
||||
ie.export(i)
|
||||
ie = JsonLinesItemExporter(output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
self.assertEqual(output.getvalue(), '{"age": 22, "name": "John"}\n')
|
||||
self.assertEqual(output.getvalue(), '{"age": "22", "name": "John"}\n')
|
||||
|
||||
"""
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue