mirror of https://github.com/scrapy/scrapy.git
rename some exporter methods and complete exporter tests refactoring
This commit is contained in:
Pablo Hoffman
parent
6b7162f3ca
commit
20e82335e2
|
|
@ -86,8 +86,8 @@ Example::
|
|||
price = Field(serializer=serialize_price)
|
||||
|
||||
|
||||
2. Overriding the serialize() method
|
||||
------------------------------------
|
||||
2. Overriding the serialize_field() method
|
||||
------------------------------------------
|
||||
|
||||
You can also override the :meth:`~BaseItemExporter.serialize` method to
|
||||
customize how your field value will be exported.
|
||||
|
|
@ -101,10 +101,10 @@ Example::
|
|||
|
||||
class ProductXmlExporter(XmlItemExporter):
|
||||
|
||||
def serialize(self, field, name, value):
|
||||
def serialize_field(self, field, name, value):
|
||||
if filed == 'price':
|
||||
return '$ %s' % str(value)
|
||||
return super(Product, self).serialize(field, name, value)
|
||||
return super(Product, self).serialize_field(field, name, value)
|
||||
|
||||
.. _topics-exporters-reference:
|
||||
|
||||
|
|
@ -129,10 +129,19 @@ BaseItemExporter
|
|||
Exports the item to the specific exporter format. This method must be
|
||||
implemented in subclasses.
|
||||
|
||||
.. method:: serialize_default(field, name, value)
|
||||
.. method:: serialize_field(field, name, value)
|
||||
|
||||
Serializes the field value to ``str``. You can override this method in
|
||||
custom Item Exporters.
|
||||
Return the serialized value for the given field. You can override this
|
||||
method (in your custom Item Exporters) if you want to control how a
|
||||
particular field or value will be serialized/exported.
|
||||
|
||||
:param field: the field being serialized
|
||||
:type field: :class:`~scrapy.item.Field` object
|
||||
|
||||
:param name: the name of the field being serialized
|
||||
:type name: str
|
||||
|
||||
:param value: the value being serialized
|
||||
|
||||
.. method:: start_exporting()
|
||||
|
||||
|
|
|
|||
|
|
@ -21,12 +21,14 @@ class BaseItemExporter(object):
|
|||
def export_item(self, item):
|
||||
raise NotImplementedError
|
||||
|
||||
def serialize(self, field, name, value):
|
||||
def serialize_field(self, field, name, value):
|
||||
serializer = field.get('serializer', identity)
|
||||
return serializer(value)
|
||||
|
||||
def _get_fields_to_export(self, item, default_value=None, include_empty=None):
|
||||
"""Return the fields to export as a list of tuples (name, value)"""
|
||||
def _get_serialized_fields(self, item, default_value=None, include_empty=None):
|
||||
"""Return the fields to export as an iterable of tuples (name,
|
||||
serialized_value)
|
||||
"""
|
||||
if include_empty is None:
|
||||
include_empty = self.export_empty_fields
|
||||
if self.fields_to_export is None:
|
||||
|
|
@ -41,7 +43,14 @@ class BaseItemExporter(object):
|
|||
nonempty_fields = set(item.keys())
|
||||
field_iter = (x for x in self.fields_to_export if x in \
|
||||
nonempty_fields)
|
||||
return [(k, item.get(k, default_value)) for k in field_iter]
|
||||
for field_name in field_iter:
|
||||
if field_name in item:
|
||||
field = item.fields[field_name]
|
||||
value = self.serialize_field(field, field_name, item[field_name])
|
||||
else:
|
||||
value = default_value
|
||||
|
||||
yield field_name, value
|
||||
|
||||
def start_exporting(self):
|
||||
pass
|
||||
|
|
@ -66,18 +75,17 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
def export_item(self, item):
|
||||
self.xg.startElement(self.item_element, {})
|
||||
for field, value in self._get_fields_to_export(item, default_value=''):
|
||||
self._export_xml_field(item.fields[field], field, value)
|
||||
for name, value in self._get_serialized_fields(item, default_value=''):
|
||||
self._export_xml_field(name, value)
|
||||
self.xg.endElement(self.item_element)
|
||||
|
||||
def finish_exporting(self):
|
||||
self.xg.endElement(self.root_element)
|
||||
self.xg.endDocument()
|
||||
|
||||
def _export_xml_field(self, field, name, value):
|
||||
def _export_xml_field(self, name, serialized_value):
|
||||
self.xg.startElement(name, {})
|
||||
if value is not None:
|
||||
self.xg.characters(self.serialize(field, name, value))
|
||||
self.xg.characters(serialized_value)
|
||||
self.xg.endElement(name)
|
||||
|
||||
|
||||
|
|
@ -97,7 +105,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self.csv_writer.writerow(self.fields_to_export)
|
||||
|
||||
def export_item(self, item):
|
||||
fields = self._get_fields_to_export(item, default_value='', \
|
||||
fields = self._get_serialized_fields(item, default_value='', \
|
||||
include_empty=True)
|
||||
|
||||
values = [x[1] for x in fields]
|
||||
|
|
@ -111,7 +119,7 @@ class PickleItemExporter(BaseItemExporter):
|
|||
self.pickler = Pickler(*args, **kwargs)
|
||||
|
||||
def export_item(self, item):
|
||||
self.pickler.dump(dict(self._get_fields_to_export(item)))
|
||||
self.pickler.dump(dict(self._get_serialized_fields(item)))
|
||||
|
||||
|
||||
class PprintItemExporter(BaseItemExporter):
|
||||
|
|
@ -121,5 +129,5 @@ class PprintItemExporter(BaseItemExporter):
|
|||
self.file = file
|
||||
|
||||
def export_item(self, item):
|
||||
itemdict = dict(self._get_fields_to_export(item))
|
||||
itemdict = dict(self._get_serialized_fields(item))
|
||||
self.file.write(pprint.pformat(itemdict) + '\n')
|
||||
|
|
|
|||
|
|
@ -13,7 +13,5 @@ class JsonLinesItemExporter(BaseItemExporter):
|
|||
self.encoder = json.JSONEncoder(*args, **kwargs)
|
||||
|
||||
def export_item(self, item):
|
||||
itemdict = {}
|
||||
for field, value in self._get_fields_to_export(item):
|
||||
itemdict[field] = self.serialize(item.fields[field], field, value)
|
||||
itemdict = dict(self._get_serialized_fields(item))
|
||||
self.file.write(self.encoder.encode(itemdict) + '\n')
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
from cPickle import Pickler
|
||||
import cPickle as pickle
|
||||
from cStringIO import StringIO
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
|
||||
PickleItemExporter, CsvItemExporter, XmlItemExporter
|
||||
|
||||
|
|
@ -13,39 +12,50 @@ class TestItem(Item):
|
|||
age = Field()
|
||||
|
||||
|
||||
class BaseTest(unittest.TestCase):
|
||||
class BaseItemExporterTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.i = TestItem(name=u'John', age='22')
|
||||
self.ie = BaseItemExporter()
|
||||
|
||||
self.output = StringIO()
|
||||
self.ie = self._get_exporter()
|
||||
|
||||
class BaseItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
self.assertRaises(NotImplementedError, self.ie.export_item, self.i)
|
||||
def _get_exporter(self):
|
||||
return BaseItemExporter()
|
||||
|
||||
def test_serialize(self):
|
||||
self.assertEqual(self.ie.serialize( \
|
||||
def _check_output(self):
|
||||
pass
|
||||
|
||||
def test_export_item(self):
|
||||
self.ie.start_exporting()
|
||||
try:
|
||||
self.ie.export_item(self.i)
|
||||
except NotImplementedError:
|
||||
if self.ie.__class__ is not BaseItemExporter:
|
||||
raise
|
||||
self.ie.finish_exporting()
|
||||
self._check_output()
|
||||
|
||||
def test_serialize_field(self):
|
||||
self.assertEqual(self.ie.serialize_field( \
|
||||
self.i.fields['name'], 'name', self.i['name']), 'John')
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(self.i.fields['age'], 'age', self.i['age']), '22')
|
||||
self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '22')
|
||||
|
||||
def test_exporter_custom_serializer(self):
|
||||
class CustomItemExporter(BaseItemExporter):
|
||||
def serialize(self, field, name, value):
|
||||
def serialize_field(self, field, name, value):
|
||||
if name == 'age':
|
||||
return str(int(value) + 1)
|
||||
else:
|
||||
return super(CustomItemExporter, self).serialize(field, \
|
||||
return super(CustomItemExporter, self).serialize_field(field, \
|
||||
name, value)
|
||||
|
||||
ie = CustomItemExporter()
|
||||
|
||||
self.assertEqual( \
|
||||
ie.serialize(self.i.fields['name'], 'name', self.i['name']), 'John')
|
||||
ie.serialize_field(self.i.fields['name'], 'name', self.i['name']), 'John')
|
||||
self.assertEqual(
|
||||
ie.serialize(self.i.fields['age'], 'age', self.i['age']), '23')
|
||||
ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '23')
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
def custom_serializer(value):
|
||||
|
|
@ -58,57 +68,44 @@ class BaseItemExporterTest(BaseTest):
|
|||
i = CustomFieldItem(name=u'John', age='22')
|
||||
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual( \
|
||||
self.ie.serialize(i.fields['age'], 'age', i['age']), '24')
|
||||
self.ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
|
||||
|
||||
def test_fields_to_export(self):
|
||||
ie = BaseItemExporter()
|
||||
ie.fields_to_export = ['name']
|
||||
|
||||
self.assertEqual(ie._get_fields_to_export(self.i), [('name', 'John')])
|
||||
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John')])
|
||||
|
||||
class PprintItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self):
|
||||
return PprintItemExporter(self.output)
|
||||
|
||||
def _check_output(self):
|
||||
self.assertEqual(dict(self.i), eval(self.output.getvalue()))
|
||||
|
||||
|
||||
class PprintItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
ie = PprintItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
self.assertEqual(self.output.getvalue(), "{'age': '22', 'name': u'John'}\n")
|
||||
|
||||
|
||||
class PickleItemExporterTest(BaseTest):
|
||||
class PickleItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def test_export(self):
|
||||
output = StringIO()
|
||||
ie = PickleItemExporter(output)
|
||||
def _get_exporter(self):
|
||||
return PickleItemExporter(self.output)
|
||||
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
def _check_output(self):
|
||||
self.assertEqual(dict(self.i), pickle.loads(self.output.getvalue()))
|
||||
|
||||
poutput = StringIO()
|
||||
p = Pickler(poutput)
|
||||
p.dump(dict(self.i))
|
||||
|
||||
self.assertEqual(output.getvalue(), poutput.getvalue())
|
||||
class CsvItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self):
|
||||
return CsvItemExporter(self.output)
|
||||
|
||||
class CsvItemExporterTest(BaseTest):
|
||||
|
||||
def test_export(self):
|
||||
ie = CsvItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
def _check_output(self):
|
||||
self.assertEqual(self.output.getvalue(), '22,John\r\n')
|
||||
|
||||
def test_header(self):
|
||||
ie = CsvItemExporter(self.output)
|
||||
output = StringIO()
|
||||
ie = CsvItemExporter(output)
|
||||
ie.include_headers_line = True
|
||||
|
||||
self.assertRaises(RuntimeError, ie.start_exporting)
|
||||
|
|
@ -118,25 +115,22 @@ class CsvItemExporterTest(BaseTest):
|
|||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\r\n')
|
||||
self.assertEqual(output.getvalue(), 'age,name\r\n22,John\r\n')
|
||||
|
||||
|
||||
class XmlItemExporterTest(BaseTest):
|
||||
class XmlItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def test_export(self):
|
||||
ie = XmlItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
def _get_exporter(self):
|
||||
return XmlItemExporter(self.output)
|
||||
|
||||
def _check_output(self):
|
||||
expected_value = '<?xml version="1.0" encoding="iso-8859-1"?>\n<items><item><age>22</age><name>John</name></item></items>'
|
||||
|
||||
self.assertEqual(self.output.getvalue(), expected_value)
|
||||
|
||||
|
||||
class JsonLinesItemExporterTest(BaseTest):
|
||||
class JsonLinesItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def setUp(self):
|
||||
def _get_exporter(self):
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
|
|
@ -144,17 +138,10 @@ class JsonLinesItemExporterTest(BaseTest):
|
|||
import simplejson
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("simplejson module not available")
|
||||
|
||||
super(JsonLinesItemExporterTest, self).setUp()
|
||||
|
||||
def test_export(self):
|
||||
from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter
|
||||
return JsonLinesItemExporter(self.output)
|
||||
|
||||
ie = JsonLinesItemExporter(self.output)
|
||||
ie.start_exporting()
|
||||
ie.export_item(self.i)
|
||||
ie.finish_exporting()
|
||||
|
||||
def _check_output(self):
|
||||
self.assertEqual(self.output.getvalue(), '{"age": "22", "name": "John"}\n')
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue