From 0ed849248ff572eb502853d5d6b8c7fe03cc11b0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 23 Aug 2009 05:48:35 -0300 Subject: [PATCH] some improvements to item exporters - passed previous class attributes to instances attributes - better handling of constructor arguments - better coverage on unittets (including encoding) - updated documentation with new changes --- docs/experimental/exporters.rst | 124 ++++++++++------ scrapy/contrib/exporter/__init__.py | 69 +++++---- scrapy/contrib/exporter/jsonlines.py | 6 +- scrapy/tests/test_contrib_exporter.py | 200 ++++++++++++++------------ 4 files changed, 231 insertions(+), 168 deletions(-) diff --git a/docs/experimental/exporters.rst b/docs/experimental/exporters.rst index 067dfa2d2..d03ec404c 100644 --- a/docs/experimental/exporters.rst +++ b/docs/experimental/exporters.rst @@ -67,6 +67,8 @@ it to the serialization library, if the exporter supports it. There are ways to customize how a field will be serialized, which are described next. +.. _topics-exporters-serializers: + 1. Declaring a serializer in the field -------------------------------------- @@ -120,9 +122,16 @@ these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8') - This is the base class for all Item Exporters, and it's an abstract class. + This is the (abstract) base class for all Item Exporters. It provides + support for common features used by all (concrete) Item Exporters, such as + defining what fields to export, whether to export empty fields, or which + encoding to use. + + These features can be configured through the constructor arguments which + populate their respective attributes: :attr:`fields_to_export`, + :attr:`export_empty_fields`, :attr:`encoding`. .. method:: export_item(item) @@ -135,6 +144,12 @@ BaseItemExporter method (in your custom Item Exporters) if you want to control how a particular field or value will be serialized/exported. + By default, this method looks for a serializer :ref:`declared in the item + field ` and returns the result of applying + that serializer to the value. If no serializer is found, it returns the + value unchanged except for ``unicode`` values which are encoded to + ``str`` using the encoding declared in the :attr:`encoding` attribute. + :param field: the field being serialized :type field: :class:`~scrapy.item.Field` object @@ -162,24 +177,39 @@ BaseItemExporter Some exporters (like :class:`CsvItemExporter`) respect the order of the fields defined in this attribute. - .. attribute:: export_empty_elements + .. attribute:: export_empty_fields - Whether to include empty elements in the exported XML (in case of - empty/missing fields). Defaults to ``False``. + Whether to include empty/unpopulated item fields in the exported data. + Defaults to ``False``. + + .. attribute:: encoding + + The encoding that will be used to encode unicode values. This only + affects unicode values (which are always serialized to str using this + encoding). Other value types are passed unchanged to the specific + serialization library. .. highlight:: none XmlItemExporter --------------- -.. class:: XmlItemExporter(file) +.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs) - Exports Items in XML format to the specified file object. You must also set - the :attr:`fields_to_export` attribute to use it. + Exports Items in XML format to the specified file object. - The default output of this exporter would be:: + :param root_element: The name of root element in the exported XML. + :type root_element: str - + :param item_element: The name of each item element in the exported XML. + :type item_element: str + + The additional keyword arguments of this constructor are passed to the + :class:`BaseItemExporter` constructor. + + A typical output of this exporter would be:: + + Color TV @@ -191,62 +221,70 @@ XmlItemExporter - .. attribute:: root_element - - The name of root element in the exported XML. Defaults to ``'items'``. - - .. attribute:: item_element - - The name of each item element in the exported XML. Defaults to ``'item'``. CsvItemExporter --------------- -.. class:: CsvItemExporter(\*args, \**kwargs) +.. class:: CsvItemExporter(file, include_headers_line=False, \**kwargs) - Exports Items in CSV format. The constructor arguments will be passed to the - `csv.writer`_ constructor. This exporter respects the order of fields in the - :attr:`BaseItemExporter.fields_to_export` attribute. + Exports Items in CSV format to the given file-like object. If the + :attr:`fields_to_export` attribute is set, it will be used to define the + CSV columns and their order. The :attr:`export_empty_fields` attribute has + no effect on this exporter. - The default output of this exporter would be:: + :param include_headers_line: If enabled, makes the exporter output a header + line with the field names taken from + :attr:`BaseItemExporter.fields_to_export` so that attribute must also be + set in order to work (otherwise it raises a :exc:`RuntimeError`) + :type include_headers_line: boolean + + The additional keyword arguments of this constructor are passed to the + :class:`BaseItemExporter` constructor, and then to the `csv.writer`_ + constructor, so you can use any `csv.writer` constructor argument to + customize this exporter. + + A typical output of this exporter would be:: Color TV,1200 DVD player,200 - .. attribute:: include_headers_line - - Makes the exporter output a header line with the field names taken from - :attr:`BaseItemExporter.fields_to_export` so that attribute must also be - set in order to work. - - Defaults to ``False``. - .. _csv.writer: http://docs.python.org/library/csv.html#csv.writer PickleItemExporter ------------------ -.. class:: PickleItemExporter(\*args, \**kwargs) +.. class:: PickleItemExporter(file, protocol=0, \**kwargs) - Exports Items in pickle format. The constructor arguments will be passed to - the `Pickler`_ constructor. This is a binary format, so no output examples - are provided. + Exports Items in pickle format to the given file-like object. + + :param protocol: The pickle protocol to use. + :type protocol: int -.. _Pickler: http://docs.python.org/library/pickle.html#pickle.Pickler + For more information, refer to the `pickle module documentation`_. + + The additional keyword arguments of this constructor are passed to the + :class:`BaseItemExporter` constructor. + + This isn't a human readable format, so no output examples are provided. + +.. _pickle module: http://docs.python.org/library/pickle.html PprintItemExporter ------------------ -.. class:: PprintItemExporter(file) +.. class:: PprintItemExporter(file, \**kwargs) Exports Items in pretty print format to the specified file object. - The default output of this exporter would be:: + The additional keyword arguments of this constructor are passed to the + :class:`BaseItemExporter` constructor. + + A typical output of this exporter would be:: {'name': 'Color TV', 'price': '1200'} {'name': 'DVD player', 'price': '200'} - Longer lines would get pretty-formatted. + Longer lines (when present) are pretty-formatted. JsonLinesItemExporter --------------------- @@ -254,11 +292,13 @@ JsonLinesItemExporter .. module:: scrapy.contrib.exporter.jsonlines :synopsis: JsonLines Item Exporter -.. class:: JsonLinesItemExporter(file, \*args, \**kwargs) +.. class:: JsonLinesItemExporter(file, \**kwargs) - Exports Items in JSON format to the specified file object, writing one - serialized item per line. The additional constructor arguments are passed to - the `JSONEncoder` constructor. + Exports Items in JSON format to the specified file-like object, writing one + JSON-encoded item per line. The additional constructor arguments are passed + to the :class:`BaseItemExporter` constructor, and to the `JSONEncoder`_ + constructor, so you can use any `JSONEncoder`_ constructor argument to + customize the exporter. The default output of this exporter would be:: diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 069373408..80dc704e4 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -11,20 +11,38 @@ from xml.sax.saxutils import XMLGenerator __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ 'CsvItemExporter', 'XmlItemExporter'] -identity = lambda x: x - class BaseItemExporter(object): - fields_to_export = None - export_empty_fields = False + def __init__(self, **kwargs): + self._configure(kwargs) + + def _configure(self, options, dont_fail=False): + """Configure the exporter by poping options from the ``options`` dict. + If dont_fail is set, it won't raise an exception on unexpected options + (useful for using with keyword arguments in subclasses constructors) + """ + self.fields_to_export = options.pop('fields_to_export', None) + self.export_empty_fields = options.pop('export_empty_fields', False) + self.encoding = options.pop('encoding', 'utf-8') + if not dont_fail and options: + raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) def export_item(self, item): raise NotImplementedError def serialize_field(self, field, name, value): - serializer = field.get('serializer', identity) + serializer = field.get('serializer', self._to_str_if_unicode) return serializer(value) + def start_exporting(self): + pass + + def finish_exporting(self): + pass + + def _to_str_if_unicode(self, value): + return value.encode(self.encoding) if isinstance(value, unicode) else value + def _get_serialized_fields(self, item, default_value=None, include_empty=None): """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -52,22 +70,14 @@ class BaseItemExporter(object): yield field_name, value - def start_exporting(self): - pass - - def finish_exporting(self): - pass - - class XmlItemExporter(BaseItemExporter): - item_element = 'item' - root_element = 'items' - - def __init__(self, file): - super(XmlItemExporter, self).__init__() - self.xg = XMLGenerator(file) + def __init__(self, file, **kwargs): + self.item_element = kwargs.pop('item_element', 'item') + self.root_element = kwargs.pop('root_element', 'items') + self._configure(kwargs) + self.xg = XMLGenerator(file, encoding=self.encoding) def start_exporting(self): self.xg.startDocument() @@ -91,17 +101,16 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - include_headers_line = False - - def __init__(self, *args, **kwargs): - super(CsvItemExporter, self).__init__() - self.csv_writer = csv.writer(*args, **kwargs) + def __init__(self, file, include_headers_line=False, **kwargs): + self._configure(kwargs, dont_fail=True) + self.include_headers_line = include_headers_line + self.csv_writer = csv.writer(file, **kwargs) def start_exporting(self): if self.include_headers_line: if not self.fields_to_export: - raise RuntimeError("You must set fields_to_export in order to" + \ - " use include_headers_line") + raise RuntimeError("You must set fields_to_export in order" + \ + " to use include_headers_line") self.csv_writer.writerow(self.fields_to_export) def export_item(self, item): @@ -114,9 +123,9 @@ class CsvItemExporter(BaseItemExporter): class PickleItemExporter(BaseItemExporter): - def __init__(self, *args, **kwargs): - super(PickleItemExporter, self).__init__() - self.pickler = Pickler(*args, **kwargs) + def __init__(self, file, protocol=0, **kwargs): + self._configure(kwargs) + self.pickler = Pickler(file, protocol) def export_item(self, item): self.pickler.dump(dict(self._get_serialized_fields(item))) @@ -124,8 +133,8 @@ class PickleItemExporter(BaseItemExporter): class PprintItemExporter(BaseItemExporter): - def __init__(self, file): - super(PprintItemExporter, self).__init__() + def __init__(self, file, **kwargs): + self._configure(kwargs) self.file = file def export_item(self, item): diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 67a567c87..4bd1b46b3 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -7,10 +7,10 @@ except ImportError: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, *args, **kwargs): - super(JsonLinesItemExporter, self).__init__() + def __init__(self, file, **kwargs): + self._configure(kwargs) self.file = file - self.encoder = json.JSONEncoder(*args, **kwargs) + self.encoder = json.JSONEncoder(**kwargs) def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index c28f89081..6a7383d96 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -4,6 +4,7 @@ from cStringIO import StringIO from twisted.trial import unittest from scrapy.item import Item, Field +from scrapy.utils.python import str_to_unicode from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -15,16 +16,21 @@ class TestItem(Item): class BaseItemExporterTest(unittest.TestCase): def setUp(self): - self.i = TestItem(name=u'John', age='22') + self.i = TestItem(name=u'John\xa3', age='22') self.output = StringIO() self.ie = self._get_exporter() - def _get_exporter(self): - return BaseItemExporter() + def _get_exporter(self, **kwargs): + return BaseItemExporter(**kwargs) def _check_output(self): pass + def _assert_expected_item(self, exported_dict): + for k, v in exported_dict.items(): + exported_dict[k] = str_to_unicode(v) + self.assertEqual(self.i, exported_dict) + def test_export_item(self): self.ie.start_exporting() try: @@ -37,10 +43,104 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): self.assertEqual(self.ie.serialize_field( \ - self.i.fields['name'], 'name', self.i['name']), 'John') + self.i.fields['name'], 'name', self.i['name']), 'John\xc2\xa3') self.assertEqual( \ self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '22') + def test_fields_to_export(self): + ie = self._get_exporter(fields_to_export=['name']) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) + + ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') + name = list(ie._get_serialized_fields(self.i))[0][1] + assert isinstance(name, str) + self.assertEqual(name, 'John\xa3') + + def test_field_custom_serializer(self): + def custom_serializer(value): + return str(int(value) + 2) + + class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + i = CustomFieldItem(name=u'John\xa3', age='22') + + ie = self._get_exporter() + self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') + self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + + +class PprintItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + return PprintItemExporter(self.output, **kwargs) + + def _check_output(self): + self._assert_expected_item(eval(self.output.getvalue())) + + +class PickleItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + return PickleItemExporter(self.output, **kwargs) + + def _check_output(self): + self._assert_expected_item(pickle.loads(self.output.getvalue())) + + +class CsvItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + return CsvItemExporter(self.output, **kwargs) + + def _check_output(self): + self.assertEqual(self.output.getvalue(), '22,John\xc2\xa3\r\n') + + def test_header(self): + output = StringIO() + ie = CsvItemExporter(output, include_headers_line=True) + self.assertRaises(RuntimeError, ie.start_exporting) + + ie = CsvItemExporter(output, include_headers_line=True, \ + fields_to_export=self.i.fields.keys()) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() + self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + + +class XmlItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + return XmlItemExporter(self.output, **kwargs) + + def _check_output(self): + expected_value = '\n22John\xc2\xa3' + self.assertEqual(self.output.getvalue(), expected_value) + + +class JsonLinesItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + try: + import json + except ImportError: + try: + import simplejson + except ImportError: + raise unittest.SkipTest("simplejson module not available") + from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter + return JsonLinesItemExporter(self.output, **kwargs) + + def _check_output(self): + import simplejson + exported = simplejson.loads(self.output.getvalue().strip()) + self.assertEqual(exported, dict(self.i)) + + +class CustomItemExporterTest(unittest.TestCase): + def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): def serialize_field(self, field, name, value): @@ -50,99 +150,13 @@ class BaseItemExporterTest(unittest.TestCase): return super(CustomItemExporter, self).serialize_field(field, \ name, value) + i = TestItem(name=u'John', age='22') ie = CustomItemExporter() self.assertEqual( \ - ie.serialize_field(self.i.fields['name'], 'name', self.i['name']), 'John') + ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') self.assertEqual( - ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '23') - - def test_field_custom_serializer(self): - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John', age='22') - - self.assertEqual( \ - self.ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual( \ - self.ie.serialize_field(i.fields['age'], 'age', i['age']), '24') - - def test_fields_to_export(self): - ie = BaseItemExporter() - ie.fields_to_export = ['name'] - - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John')]) - -class PprintItemExporterTest(BaseItemExporterTest): - - def _get_exporter(self): - return PprintItemExporter(self.output) - - def _check_output(self): - self.assertEqual(dict(self.i), eval(self.output.getvalue())) - - -class PickleItemExporterTest(BaseItemExporterTest): - - def _get_exporter(self): - return PickleItemExporter(self.output) - - def _check_output(self): - self.assertEqual(dict(self.i), pickle.loads(self.output.getvalue())) - -class CsvItemExporterTest(BaseItemExporterTest): - - def _get_exporter(self): - return CsvItemExporter(self.output) - - def _check_output(self): - self.assertEqual(self.output.getvalue(), '22,John\r\n') - - def test_header(self): - output = StringIO() - ie = CsvItemExporter(output) - ie.include_headers_line = True - - self.assertRaises(RuntimeError, ie.start_exporting) - - ie.fields_to_export = self.i.fields.keys() - ie.start_exporting() - ie.export_item(self.i) - ie.finish_exporting() - - self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n') - - -class XmlItemExporterTest(BaseItemExporterTest): - - def _get_exporter(self): - return XmlItemExporter(self.output) - - def _check_output(self): - expected_value = '\n22John' - self.assertEqual(self.output.getvalue(), expected_value) - - -class JsonLinesItemExporterTest(BaseItemExporterTest): - - def _get_exporter(self): - try: - import json - except ImportError: - try: - import simplejson - except ImportError: - raise unittest.SkipTest("simplejson module not available") - from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter - return JsonLinesItemExporter(self.output) - - def _check_output(self): - self.assertEqual(self.output.getvalue(), '{"age": "22", "name": "John"}\n') + ie.serialize_field(i.fields['age'], 'age', i['age']), '23') if __name__ == '__main__':