From 314e8dea4303bf8a7d92a484bb9438116c8a08f1 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Wed, 19 Aug 2009 19:05:33 -0300 Subject: [PATCH] added new item exporter tests, introduced some api changes --- docs/experimental/exporters.rst | 34 ++++-- scrapy/contrib/exporter/__init__.py | 28 +++-- scrapy/contrib/exporter/jsonlines.py | 2 +- scrapy/tests/test_contrib_exporter.py | 148 +++++++++++++++----------- 4 files changed, 127 insertions(+), 85 deletions(-) diff --git a/docs/experimental/exporters.rst b/docs/experimental/exporters.rst index 7f7f4ad27..6916884e3 100644 --- a/docs/experimental/exporters.rst +++ b/docs/experimental/exporters.rst @@ -20,8 +20,11 @@ Using Item Exporters In order to use a Item Exporter, you must instantiate it with its required args. Different exporters require different args, so check each exporter documentation to be sure, in :ref:`topics-exporters-reference`. After you have -instantiated you exporter, you have to call the -:meth:`~BaseItemExporter.export` method with each item you want to export. +instantiated you exporter, you call the method +:meth:`~BaseItemExporter.start_exporting` in order to initialize the exporting +proces, then you call :meth:`~BaseItemExporter.export_item` method for each +item you want to export, and finally call +:meth:`~BaseItemExporter.finish_exporting` to finalize the exporting process. Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline `:: @@ -38,13 +41,14 @@ Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline def domain_open(self, domain): self.file = open('%s_products.xml' % domain) self.exporter = XmlItemExporter(self.file) + self.exporter.start_exporting() def domain_closed(self, domain): - self.exporter.close() + self.exporter.finish_exporting() self.file.close() def process_item(self, domain, item): - self.exporter.export(item) + self.exporter.export_item(item) return item @@ -120,7 +124,7 @@ BaseItemExporter This is the base class for all Item Exporters, and it's an abstract class. - .. method:: export(item) + .. method:: export_item(item) Exports the item to the specific exporter format. This method must be implemented in subclasses. @@ -130,10 +134,16 @@ BaseItemExporter Serializes the field value to ``str``. You can override this method in custom Item Exporters. - .. method:: close() + .. method:: start_exporting() - Called when there are no more items to export, so the exporter can close - the serialization, for those formats that require it (like XML). + Makes the exporter initialize the export process, in here exporters may + output information required by the exporter's format. + + .. method:: finish_exporting() + + You must call it when there are no more items to export, so the exporter + can close the serialization output, for those formats that require it + (like XML). .. attribute:: fields_to_export @@ -196,9 +206,11 @@ CsvItemExporter .. attribute:: include_headers_line - If ``True`` the first line in the CSV export will include the name of the - fields columns, taken from the :attr:`BaseItemExporter.fields_to_export` - attribute. Defaults to ``False``. + Makes the exporter output a header line with the field names taken from + :attr:`BaseItemExporter.fields_to_export` so that attribute must also be + set in order to work. + + Defaults to ``False``. .. _csv.writer: http://docs.python.org/library/csv.html#csv.writer diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index e1818529c..38cd732b1 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -18,7 +18,7 @@ class BaseItemExporter(object): fields_to_export = None export_empty_fields = False - def export(self, item): + def export_item(self, item): raise NotImplementedError def serialize(self, field, name, value): @@ -43,9 +43,13 @@ class BaseItemExporter(object): nonempty_fields) return [(k, item.get(k, default_value)) for k in field_iter] - def close(self): + def start_exporting(self): pass + def finish_exporting(self): + pass + + class XmlItemExporter(BaseItemExporter): @@ -55,15 +59,18 @@ class XmlItemExporter(BaseItemExporter): def __init__(self, file): super(XmlItemExporter, self).__init__() self.xg = XMLGenerator(file) + + def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) - def export(self, item): + def export_item(self, item): self.xg.startElement(self.item_element, {}) for field, value in self._get_fields_to_export(item, default_value=''): self._export_xml_field(item.fields[field], field, value) + self.xg.endElement(self.item_element) - def close(self): + def finish_exporting(self): self.xg.endElement(self.root_element) self.xg.endDocument() @@ -81,15 +88,18 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, *args, **kwargs): super(CsvItemExporter, self).__init__() self.csv_writer = csv.writer(*args, **kwargs) + + def start_exporting(self): if self.include_headers_line: if not self.fields_to_export: - raise RuntimeError("To use include_headers_line you must " \ - "define fields_to_export attribute") + raise RuntimeError("You must set fields_to_export in order to" + \ + " use include_headers_line") self.csv_writer.writerow(self.fields_to_export) - def export(self, item): + def export_item(self, item): fields = self._get_fields_to_export(item, default_value='', \ include_empty=True) + values = [x[1] for x in fields] self.csv_writer.writerow(values) @@ -100,7 +110,7 @@ class PickleItemExporter(BaseItemExporter): super(PickleItemExporter, self).__init__() self.pickler = Pickler(*args, **kwargs) - def export(self, item): + def export_item(self, item): self.pickler.dump(dict(self._get_fields_to_export(item))) @@ -110,6 +120,6 @@ class PprintItemExporter(BaseItemExporter): super(PprintItemExporter, self).__init__() self.file = file - def export(self, item): + def export_item(self, item): itemdict = dict(self._get_fields_to_export(item)) self.file.write(pprint.pformat(itemdict) + '\n') diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 4651b0b5f..f19ec4cf6 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -14,4 +14,4 @@ class JsonLinesItemExporter(BaseItemExporter): def export(self, item): itemdict = dict(self._get_fields_to_export(item)) - self.file.write(self.encoder(itemdict) + '\n') + self.file.write(self.encoder.encode(itemdict) + '\n') diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index e190f610c..09c8a397b 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -5,8 +5,6 @@ from twisted.trial import unittest from scrapy.newitem import Item, Field -# FIXME: fix tests -""" from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -15,108 +13,128 @@ class TestItem(Item): age = Field() -class BaseItemExporterTest(unittest.TestCase): - +class BaseTest(unittest.TestCase): + def setUp(self): + self.i = TestItem(name=u'John', age='22') + self.ie = BaseItemExporter() + + self.output = StringIO() + +class BaseItemExporterTest(BaseTest): + def test_export(self): - i = TestItem(name=u'John', age=22) + self.assertRaises(NotImplementedError, self.ie.export_item, self.i) - ie = BaseItemExporter() - - self.assertRaises(NotImplementedError, ie.export, i) - - def test_default_serializer(self): - i = TestItem(name=u'John', age=22) - - ie = BaseItemExporter() - - self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual( ie._serialize_field(i.fields['age'], 'age', i['age']), '22') + def test_serialize(self): + self.assertEqual(self.ie.serialize( \ + self.i.fields['name'], 'name', self.i['name']), 'John') + self.assertEqual( \ + self.ie.serialize(self.i.fields['age'], 'age', self.i['age']), '22') def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): - def serialize_age(self, field, name, value): - return str(value + 1) - - i = TestItem(name=u'John', age=22) + def serialize(self, field, name, value): + if name == 'age': + return str(int(value) + 1) + else: + return super(CustomItemExporter, self).serialize(field, \ + name, value) ie = CustomItemExporter() - self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '23') + self.assertEqual( \ + ie.serialize(self.i.fields['name'], 'name', self.i['name']), 'John') + self.assertEqual( + ie.serialize(self.i.fields['age'], 'age', self.i['age']), '23') def test_field_custom_serializer(self): - class CustomField(Field): - def serializer(self, field, name, value): - return str(value + 2) + def custom_serializer(value): + return str(int(value) + 2) class CustomFieldItem(Item): name = Field() - age = CustomField() + age = Field(serializer=custom_serializer) - i = CustomFieldItem(name=u'John', age=22) + i = CustomFieldItem(name=u'John', age='22') + self.assertEqual( \ + self.ie.serialize(i.fields['name'], 'name', i['name']), 'John') + self.assertEqual( \ + self.ie.serialize(i.fields['age'], 'age', i['age']), '24') + + def test_fields_to_export(self): ie = BaseItemExporter() + ie.fields_to_export = ['name'] - self.assertEqual(ie._serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie._serialize_field(i.fields['age'], 'age', i['age']), '24') + self.assertEqual(ie._get_fields_to_export(self.i), [('name', 'John')]) -class PprintItemExporterTest(unittest.TestCase): +class PprintItemExporterTest(BaseTest): def test_export(self): - i = TestItem(name=u'John', age=22) + ie = PprintItemExporter(self.output) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() - output = StringIO() - ie = PprintItemExporter(output) - ie.export(i) - - self.assertEqual(output.getvalue(), "{'age': 22, 'name': u'John'}\n") + self.assertEqual(self.output.getvalue(), "{'age': '22', 'name': u'John'}\n") -class PickleItemExporterTest(unittest.TestCase): +class PickleItemExporterTest(BaseTest): def test_export(self): - i = TestItem(name=u'John', age=22) - output = StringIO() ie = PickleItemExporter(output) - ie.export(i) + + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() poutput = StringIO() p = Pickler(poutput) - p.dump(dict(i)) + p.dump(dict(self.i)) self.assertEqual(output.getvalue(), poutput.getvalue()) -class CsvItemExporterTest(unittest.TestCase): +class CsvItemExporterTest(BaseTest): def test_export(self): - i = TestItem(name=u'John', age=22) + ie = CsvItemExporter(self.output) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() - output = StringIO() - ie = CsvItemExporter(output) - ie.fields_to_export = i.fields.keys() - ie.export(i) + self.assertEqual(self.output.getvalue(), '22,John\r\n') - self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n') + def test_header(self): + ie = CsvItemExporter(self.output) + ie.include_headers_line = True + + self.assertRaises(RuntimeError, ie.start_exporting) + + ie.fields_to_export = self.i.fields.keys() + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() + + self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\r\n') -class XmlItemExporterTest(unittest.TestCase): +class XmlItemExporterTest(BaseTest): def test_export(self): - i = TestItem(name=u'John', age=22) + ie = XmlItemExporter(self.output) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() - output = StringIO() - ie = XmlItemExporter(output) - ie.fields_to_export = i.fields.keys() - ie.export(i) + expected_value = '\n22John' - self.assertEqual(output.getvalue(), \ - '\n22John') + self.assertEqual(self.output.getvalue(), expected_value) -class JSONItemExporterTest(unittest.TestCase): +class JSONItemExporterTest(BaseTest): def setUp(self): try: @@ -127,17 +145,19 @@ class JSONItemExporterTest(unittest.TestCase): except ImportError: raise unittest.SkipTest("simplejson module not available") + from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter + + super(JSONItemExporterTest, self).__init__() + def test_export(self): - from scrapy.contrib.exporter.jsonexporter import JSONItemExporter - output = StringIO() - ie = JSONItemExporter(output) - i = TestItem(name=u'John', age=22) - ie.export(i) + ie = JsonLinesItemExporter(output) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() - self.assertEqual(output.getvalue(), '{"age": 22, "name": "John"}\n') + self.assertEqual(output.getvalue(), '{"age": "22", "name": "John"}\n') -""" if __name__ == '__main__': unittest.main()