From c83fb1c7d123bc510f11ed2f75d424f2b5c8a446 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Wed, 12 Aug 2009 10:17:06 -0300 Subject: [PATCH] added newitem exporter tests and fixed exporter errors --- scrapy/newitem/exporters/__init__.py | 15 +-- scrapy/tests/test_itemexporters.py | 131 +++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 scrapy/tests/test_itemexporters.py diff --git a/scrapy/newitem/exporters/__init__.py b/scrapy/newitem/exporters/__init__.py index da6d1bd9b..9fcb996fe 100644 --- a/scrapy/newitem/exporters/__init__.py +++ b/scrapy/newitem/exporters/__init__.py @@ -22,15 +22,15 @@ class BaseItemExporter(object): def _serialize_field(self, field, name, value): if hasattr(self, 'serialize_%s' % name): - serializer = getattr('serialize_%s' % name) + serializer = getattr(self, 'serialize_%s' % name) elif hasattr(field, 'serializer'): serializer = field.serializer else: - serializer = _default_serializer(field, name, value) + serializer = self._default_serializer return serializer(field, name, value) - def _default_serializer(field, name, value): + def _default_serializer(self, field, name, value): return str(value) @@ -67,8 +67,8 @@ class CsvItemExporter(BaseItemExporter): values = [] for field in self.fields_to_export: if field in item: - values.append(self._serialize_field(item.fields[field]), field, \ - item[field]) + values.append(self._serialize_field(item.fields[field], field, + item[field])) else: values.append('') self.csv_writer.writerow(values) @@ -92,9 +92,7 @@ class XmlItemExporter(BaseItemExporter): self.xg.startElement(self.item_element, {}) for field in self.fields_to_export: if field in item: - self.xg.startElement(self.item_element, {}) self._export_xml_field(item.fields[field], field, item[field]) - self.xg.endElement(self.item_element) elif self.include_empty_elements: self.xg.startElement(self.item_element, {}) self.xg.endElement() @@ -105,9 +103,6 @@ class XmlItemExporter(BaseItemExporter): self.xg.endDocument() def _export_xml_field(self, field, name, value): - if isinstance(field, fields.ListField): - for v in value: - self._export_xml_field(field._field, 'value', v) self.xg.startElement(name, {}) self.xg.characters(self._serialize_field(field, name, value)) self.xg.endElement(name) diff --git a/scrapy/tests/test_itemexporters.py b/scrapy/tests/test_itemexporters.py new file mode 100644 index 000000000..e41ab775e --- /dev/null +++ b/scrapy/tests/test_itemexporters.py @@ -0,0 +1,131 @@ +from cPickle import Pickler +from cStringIO import StringIO +import pprint +from twisted.trial import unittest + +from scrapy.newitem import Item, Field +from scrapy.newitem.exporters import * + +class TestItem(Item): + name = Field() + age = Field() + + +class BaseItemExporterTest(unittest.TestCase): + + def test_export(self): + i = TestItem(name=u'John', age=22) + + ie = BaseItemExporter() + + self.assertRaises(NotImplementedError, ie.export, i) + + def test_default_serializer(self): + i = TestItem(name=u'John', age=22) + + ie = BaseItemExporter() + + self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') + self.assertEqual( ie._serialize_field(i.fields['age'], 'age', i['age']), '22') + + def test_exporter_custom_serializer(self): + class CustomItemExporter(BaseItemExporter): + def serialize_age(self, field, name, value): + return str(value + 1) + + i = TestItem(name=u'John', age=22) + + ie = CustomItemExporter() + + self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') + self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '23') + + def test_field_custom_serializer(self): + class CustomField(Field): + def serializer(self, field, name, value): + return str(value + 2) + + class CustomFieldItem(Item): + name = Field() + age = CustomField() + + i = CustomFieldItem(name=u'John', age=22) + + ie = BaseItemExporter() + + self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') + self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '24') + + +class PprintItemExporterTest(unittest.TestCase): + + def test_export(self): + i = TestItem(name=u'John', age=22) + + output = StringIO() + ie = PprintItemExporter(output) + ie.export(i) + + self.assertEqual(output.getvalue(), "{'age': 22, 'name': u'John'}\n") + + +class PickleItemExporterTest(unittest.TestCase): + + def test_export(self): + i = TestItem(name=u'John', age=22) + + output = StringIO() + ie = PickleItemExporter(output) + ie.export(i) + + poutput = StringIO() + p = Pickler(poutput) + p.dump(dict(i)) + + self.assertEqual(output.getvalue(), poutput.getvalue()) + + +class CsvItemExporterTest(unittest.TestCase): + + def test_export(self): + i = TestItem(name=u'John', age=22) + + output = StringIO() + ie = CsvItemExporter(output) + ie.fields_to_export = i.fields.keys() + ie.export(i) + + self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n') + + +class XmlItemExporterTest(unittest.TestCase): + + def test_export(self): + i = TestItem(name=u'John', age=22) + + output = StringIO() + ie = XmlItemExporter(output) + ie.fields_to_export = i.fields.keys() + ie.export(i) + + self.assertEqual(output.getvalue(), '\n22John') + + +class JSONItemExporterTest(unittest.TestCase): + + def setUp(self): + try: + from scrapy.newitem.exporters.jsonexporter import JSONItemExporter + self.output = StringIO() + self.ie = JSONItemExporter(self.output) + + except ImportError, e: + raise unittest.SkipTest("Json library not available") + + def test_export(self): + i = TestItem(name=u'John', age=22) + + self.ie.export(i) + + self.assertEqual(self.output.getvalue(), '{"age": 22, "name": "John"}\n') +