From 20e82335e2127448809b7a30d0de484602cae23e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 20 Aug 2009 14:09:24 -0300 Subject: [PATCH] rename some exporter methods and complete exporter tests refactoring --- docs/experimental/exporters.rst | 23 +++-- scrapy/contrib/exporter/__init__.py | 32 ++++--- scrapy/contrib/exporter/jsonlines.py | 4 +- scrapy/tests/test_contrib_exporter.py | 127 ++++++++++++-------------- 4 files changed, 94 insertions(+), 92 deletions(-) diff --git a/docs/experimental/exporters.rst b/docs/experimental/exporters.rst index 9fd0ad8c3..067dfa2d2 100644 --- a/docs/experimental/exporters.rst +++ b/docs/experimental/exporters.rst @@ -86,8 +86,8 @@ Example:: price = Field(serializer=serialize_price) -2. Overriding the serialize() method ------------------------------------- +2. Overriding the serialize_field() method +------------------------------------------ You can also override the :meth:`~BaseItemExporter.serialize` method to customize how your field value will be exported. @@ -101,10 +101,10 @@ Example:: class ProductXmlExporter(XmlItemExporter): - def serialize(self, field, name, value): + def serialize_field(self, field, name, value): if filed == 'price': return '$ %s' % str(value) - return super(Product, self).serialize(field, name, value) + return super(Product, self).serialize_field(field, name, value) .. _topics-exporters-reference: @@ -129,10 +129,19 @@ BaseItemExporter Exports the item to the specific exporter format. This method must be implemented in subclasses. - .. method:: serialize_default(field, name, value) + .. method:: serialize_field(field, name, value) - Serializes the field value to ``str``. You can override this method in - custom Item Exporters. + Return the serialized value for the given field. You can override this + method (in your custom Item Exporters) if you want to control how a + particular field or value will be serialized/exported. + + :param field: the field being serialized + :type field: :class:`~scrapy.item.Field` object + + :param name: the name of the field being serialized + :type name: str + + :param value: the value being serialized .. method:: start_exporting() diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 38cd732b1..069373408 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -21,12 +21,14 @@ class BaseItemExporter(object): def export_item(self, item): raise NotImplementedError - def serialize(self, field, name, value): + def serialize_field(self, field, name, value): serializer = field.get('serializer', identity) return serializer(value) - def _get_fields_to_export(self, item, default_value=None, include_empty=None): - """Return the fields to export as a list of tuples (name, value)""" + def _get_serialized_fields(self, item, default_value=None, include_empty=None): + """Return the fields to export as an iterable of tuples (name, + serialized_value) + """ if include_empty is None: include_empty = self.export_empty_fields if self.fields_to_export is None: @@ -41,7 +43,14 @@ class BaseItemExporter(object): nonempty_fields = set(item.keys()) field_iter = (x for x in self.fields_to_export if x in \ nonempty_fields) - return [(k, item.get(k, default_value)) for k in field_iter] + for field_name in field_iter: + if field_name in item: + field = item.fields[field_name] + value = self.serialize_field(field, field_name, item[field_name]) + else: + value = default_value + + yield field_name, value def start_exporting(self): pass @@ -66,18 +75,17 @@ class XmlItemExporter(BaseItemExporter): def export_item(self, item): self.xg.startElement(self.item_element, {}) - for field, value in self._get_fields_to_export(item, default_value=''): - self._export_xml_field(item.fields[field], field, value) + for name, value in self._get_serialized_fields(item, default_value=''): + self._export_xml_field(name, value) self.xg.endElement(self.item_element) def finish_exporting(self): self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, field, name, value): + def _export_xml_field(self, name, serialized_value): self.xg.startElement(name, {}) - if value is not None: - self.xg.characters(self.serialize(field, name, value)) + self.xg.characters(serialized_value) self.xg.endElement(name) @@ -97,7 +105,7 @@ class CsvItemExporter(BaseItemExporter): self.csv_writer.writerow(self.fields_to_export) def export_item(self, item): - fields = self._get_fields_to_export(item, default_value='', \ + fields = self._get_serialized_fields(item, default_value='', \ include_empty=True) values = [x[1] for x in fields] @@ -111,7 +119,7 @@ class PickleItemExporter(BaseItemExporter): self.pickler = Pickler(*args, **kwargs) def export_item(self, item): - self.pickler.dump(dict(self._get_fields_to_export(item))) + self.pickler.dump(dict(self._get_serialized_fields(item))) class PprintItemExporter(BaseItemExporter): @@ -121,5 +129,5 @@ class PprintItemExporter(BaseItemExporter): self.file = file def export_item(self, item): - itemdict = dict(self._get_fields_to_export(item)) + itemdict = dict(self._get_serialized_fields(item)) self.file.write(pprint.pformat(itemdict) + '\n') diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 285577a29..67a567c87 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -13,7 +13,5 @@ class JsonLinesItemExporter(BaseItemExporter): self.encoder = json.JSONEncoder(*args, **kwargs) def export_item(self, item): - itemdict = {} - for field, value in self._get_fields_to_export(item): - itemdict[field] = self.serialize(item.fields[field], field, value) + itemdict = dict(self._get_serialized_fields(item)) self.file.write(self.encoder.encode(itemdict) + '\n') diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 9d11894a3..c28f89081 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,10 +1,9 @@ -from cPickle import Pickler +import cPickle as pickle from cStringIO import StringIO from twisted.trial import unittest from scrapy.item import Item, Field - from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -13,39 +12,50 @@ class TestItem(Item): age = Field() -class BaseTest(unittest.TestCase): +class BaseItemExporterTest(unittest.TestCase): + def setUp(self): self.i = TestItem(name=u'John', age='22') - self.ie = BaseItemExporter() - self.output = StringIO() + self.ie = self._get_exporter() -class BaseItemExporterTest(BaseTest): - - def test_export(self): - self.assertRaises(NotImplementedError, self.ie.export_item, self.i) + def _get_exporter(self): + return BaseItemExporter() - def test_serialize(self): - self.assertEqual(self.ie.serialize( \ + def _check_output(self): + pass + + def test_export_item(self): + self.ie.start_exporting() + try: + self.ie.export_item(self.i) + except NotImplementedError: + if self.ie.__class__ is not BaseItemExporter: + raise + self.ie.finish_exporting() + self._check_output() + + def test_serialize_field(self): + self.assertEqual(self.ie.serialize_field( \ self.i.fields['name'], 'name', self.i['name']), 'John') self.assertEqual( \ - self.ie.serialize(self.i.fields['age'], 'age', self.i['age']), '22') + self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '22') def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): - def serialize(self, field, name, value): + def serialize_field(self, field, name, value): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize(field, \ + return super(CustomItemExporter, self).serialize_field(field, \ name, value) ie = CustomItemExporter() self.assertEqual( \ - ie.serialize(self.i.fields['name'], 'name', self.i['name']), 'John') + ie.serialize_field(self.i.fields['name'], 'name', self.i['name']), 'John') self.assertEqual( - ie.serialize(self.i.fields['age'], 'age', self.i['age']), '23') + ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '23') def test_field_custom_serializer(self): def custom_serializer(value): @@ -58,57 +68,44 @@ class BaseItemExporterTest(BaseTest): i = CustomFieldItem(name=u'John', age='22') self.assertEqual( \ - self.ie.serialize(i.fields['name'], 'name', i['name']), 'John') + self.ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') self.assertEqual( \ - self.ie.serialize(i.fields['age'], 'age', i['age']), '24') + self.ie.serialize_field(i.fields['age'], 'age', i['age']), '24') def test_fields_to_export(self): ie = BaseItemExporter() ie.fields_to_export = ['name'] - self.assertEqual(ie._get_fields_to_export(self.i), [('name', 'John')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John')]) + +class PprintItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self): + return PprintItemExporter(self.output) + + def _check_output(self): + self.assertEqual(dict(self.i), eval(self.output.getvalue())) -class PprintItemExporterTest(BaseTest): - - def test_export(self): - ie = PprintItemExporter(self.output) - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() - - self.assertEqual(self.output.getvalue(), "{'age': '22', 'name': u'John'}\n") - - -class PickleItemExporterTest(BaseTest): +class PickleItemExporterTest(BaseItemExporterTest): - def test_export(self): - output = StringIO() - ie = PickleItemExporter(output) + def _get_exporter(self): + return PickleItemExporter(self.output) - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() + def _check_output(self): + self.assertEqual(dict(self.i), pickle.loads(self.output.getvalue())) - poutput = StringIO() - p = Pickler(poutput) - p.dump(dict(self.i)) - - self.assertEqual(output.getvalue(), poutput.getvalue()) +class CsvItemExporterTest(BaseItemExporterTest): + def _get_exporter(self): + return CsvItemExporter(self.output) -class CsvItemExporterTest(BaseTest): - - def test_export(self): - ie = CsvItemExporter(self.output) - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() - + def _check_output(self): self.assertEqual(self.output.getvalue(), '22,John\r\n') def test_header(self): - ie = CsvItemExporter(self.output) + output = StringIO() + ie = CsvItemExporter(output) ie.include_headers_line = True self.assertRaises(RuntimeError, ie.start_exporting) @@ -118,25 +115,22 @@ class CsvItemExporterTest(BaseTest): ie.export_item(self.i) ie.finish_exporting() - self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\r\n') + self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n') -class XmlItemExporterTest(BaseTest): +class XmlItemExporterTest(BaseItemExporterTest): - def test_export(self): - ie = XmlItemExporter(self.output) - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() + def _get_exporter(self): + return XmlItemExporter(self.output) + def _check_output(self): expected_value = '\n22John' - self.assertEqual(self.output.getvalue(), expected_value) -class JsonLinesItemExporterTest(BaseTest): +class JsonLinesItemExporterTest(BaseItemExporterTest): - def setUp(self): + def _get_exporter(self): try: import json except ImportError: @@ -144,17 +138,10 @@ class JsonLinesItemExporterTest(BaseTest): import simplejson except ImportError: raise unittest.SkipTest("simplejson module not available") - - super(JsonLinesItemExporterTest, self).setUp() - - def test_export(self): from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter + return JsonLinesItemExporter(self.output) - ie = JsonLinesItemExporter(self.output) - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() - + def _check_output(self): self.assertEqual(self.output.getvalue(), '{"age": "22", "name": "John"}\n')