use ScrapyJSONEncoder JsonItemExporter & JsonLinesItemExporter, to support nested items properly

docs/news.rst

@@ -11,6 +11,7 @@ Scrapy changes:
 - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm``
 - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
 - StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`)
+- nested items now fully supported in JSON and JSONLines exporters
 - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider
 - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that mdule
 - dropped support for Python 2.5. See http://blog.scrapy.org/scrapy-dropping-support-for-python-25

scrapy/contrib/exporter/__init__.py

@@ -8,6 +8,7 @@ import marshal
 import json
 import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
+from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \
@@ -79,7 +80,7 @@ class JsonLinesItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs)
         self.file = file
-        self.encoder = json.JSONEncoder(**kwargs)
+        self.encoder = ScrapyJSONEncoder(**kwargs)
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
@@ -91,7 +92,7 @@ class JsonItemExporter(JsonLinesItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs)
         self.file = file
-        self.encoder = json.JSONEncoder(**kwargs)
+        self.encoder = ScrapyJSONEncoder(**kwargs)
         self.first_item = True
 
     def start_exporting(self):

scrapy/tests/test_contrib_exporter.py

@@ -173,6 +173,8 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
+    _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+
     def _get_exporter(self, **kwargs):
         return JsonLinesItemExporter(self.output, **kwargs)
 
@@ -180,8 +182,20 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         exported = json.loads(self.output.getvalue().strip())
         self.assertEqual(exported, dict(self.i))
 
+    def test_nested_item(self):
+        i1 = TestItem(name=u'Joseph\xa3', age='22')
+        i2 = TestItem(name=u'Maria', age=i1)
+        i3 = TestItem(name=u'Jesus', age=i2)
+        self.ie.start_exporting()
+        self.ie.export_item(i3)
+        self.ie.finish_exporting()
+        exported = json.loads(self.output.getvalue())
+        self.assertEqual(exported, self._expected_nested)
+
 class JsonItemExporterTest(JsonLinesItemExporterTest):
 
+    _expected_nested = [JsonLinesItemExporterTest._expected_nested]
+
     def _get_exporter(self, **kwargs):
         return JsonItemExporter(self.output, **kwargs)
 
@@ -197,6 +211,17 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         exported = json.loads(self.output.getvalue())
         self.assertEqual(exported, [dict(self.i), dict(self.i)])
 
+    def test_nested_item(self):
+        i1 = TestItem(name=u'Joseph\xa3', age='22')
+        i2 = TestItem(name=u'Maria', age=i1)
+        i3 = TestItem(name=u'Jesus', age=i2)
+        self.ie.start_exporting()
+        self.ie.export_item(i3)
+        self.ie.finish_exporting()
+        exported = json.loads(self.output.getvalue())
+        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
+        self.assertEqual(exported, [expected])
+
 class CustomItemExporterTest(unittest.TestCase):
 
     def test_exporter_custom_serializer(self):

scrapy/utils/serialize.py

@@ -7,6 +7,7 @@ from twisted.internet import defer
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
+from scrapy.item import BaseItem
 
 
 class SpiderReferencer(object):
@@ -98,6 +99,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         elif isinstance(o, defer.Deferred):
             return str(o)
+        elif isinstance(o, BaseItem):
+            return dict(o)
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
         elif isinstance(o, Response):