Added JSON item exporter with doc and unittests (closes #192), and also:

* put all json exporters in scrapy.contrib.exporters and deprecated
  scrapy.contrib.exporters.jsonlines to reduce module nesting
* use JSON exporter with EXPORT_FORMAT=json in file export pipeline

docs/faq.rst

@@ -160,3 +160,10 @@ Amazon EC2 <http://dev.scrapy.org/wiki/AmazonEC2>`_.
 .. _twistd: http://twistedmatrix.com/documents/current/core/howto/basics.html#auto1
 .. _official APT repos: http://dev.scrapy.org/wiki/AptRepos
 .. _upstart: http://upstart.ubuntu.com/
+
+Can I use JSON for large exports?
+---------------------------------
+
+It'll depend on how large your output is. See :ref:`this warning
+<json-with-large-data>` in :class:`~scrapy.contrib.exporter.JsonItemExporter`
+documentation.

docs/topics/exporters.rst

@@ -332,11 +332,38 @@ PprintItemExporter
 
    Longer lines (when present) are pretty-formatted.
 
-JsonLinesItemExporter
+JsonItemExporter
 ---------------------
 
-.. module:: scrapy.contrib.exporter.jsonlines
-   :synopsis: JsonLines Item Exporter
+.. class:: JsonItemExporter(file, \**kwargs)
+
+   Exports Items in JSON format to the specified file-like object, writing all
+   objects as a list of objects. The additional constructor arguments are
+   passed to the :class:`BaseItemExporter` constructor, and the leftover
+   arguments to the `JSONEncoder`_ constructor, so you can use any
+   `JSONEncoder`_ constructor argument to customize this exporter.
+
+   :param file: the file-like object to use for exporting the data.
+
+   A typical output of this exporter would be::
+
+        [{"name": "Color TV", "price": "1200"},
+        {"name": "DVD player", "price": "200"}]
+
+   .. _json-with-large-data:
+
+   .. warning:: JSON is very simple and flexible serialization format, but it
+      doesn't scale well for large amounts of data since incremental (aka.
+      stream-mode) parsing is not well supported (if at all) among JSON parsers
+      (on any language), and most of them just parse the entire object in
+      memory. If you want the power and simplicity of JSON with a more
+      stream-friendly format, consider using :class:`JsonLinesItemExporter`
+      instead, or splitting the output in multiple chunks.
+
+.. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder
+
+JsonLinesItemExporter
+---------------------
 
 .. class:: JsonLinesItemExporter(file, \**kwargs)
 
@@ -353,4 +380,7 @@ JsonLinesItemExporter
         {"name": "Color TV", "price": "1200"}
         {"name": "DVD player", "price": "200"}
 
+   Unlike the one produced by :class:`JsonItemExporter`, the format produced by
+   this exporter is well suited for serializing large amounts of data.
+
 .. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder

docs/topics/item-pipeline.rst

@@ -178,7 +178,9 @@ on the respective Item Exporter to get more info.
   the first line. This format requires you to specify the fields to export
   using the :setting:`EXPORT_FIELDS` setting.
 
-* ``json``: uses a :class:`~jsonlines.JsonLinesItemExporter`
+* ``json``: uses a :class:`JsonItemExporter`
+
+* ``jsonlines``: uses a :class:`JsonLinesItemExporter`
 
 * ``pickle``: uses a :class:`PickleItemExporter`
 

scrapy/contrib/exporter/__init__.py

@@ -7,9 +7,12 @@ import pprint
 from cPickle import Pickler
 from xml.sax.saxutils import XMLGenerator
 
+from scrapy.utils.py26 import json
+
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \
-    'CsvItemExporter', 'XmlItemExporter']
+    'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \
+    'JsonItemExporter']
 
 class BaseItemExporter(object):
 
@@ -71,6 +74,41 @@ class BaseItemExporter(object):
             yield field_name, value
 
 
+class JsonLinesItemExporter(BaseItemExporter):
+
+    def __init__(self, file, **kwargs):
+        self._configure(kwargs)
+        self.file = file
+        self.encoder = json.JSONEncoder(**kwargs)
+
+    def export_item(self, item):
+        itemdict = dict(self._get_serialized_fields(item))
+        self.file.write(self.encoder.encode(itemdict) + '\n')
+
+
+class JsonItemExporter(JsonLinesItemExporter):
+
+    def __init__(self, file, **kwargs):
+        self._configure(kwargs)
+        self.file = file
+        self.encoder = json.JSONEncoder(**kwargs)
+        self.first_item = True
+
+    def start_exporting(self):
+        self.file.write("[")
+
+    def finish_exporting(self):
+        self.file.write("]")
+
+    def export_item(self, item):
+        if self.first_item:
+            self.first_item = False
+        else:
+            self.file.write(',\n')
+        itemdict = dict(self._get_serialized_fields(item))
+        self.file.write(self.encoder.encode(itemdict))
+
+
 class XmlItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):

scrapy/contrib/exporter/jsonlines.py

@@ -1,13 +1,5 @@
-from scrapy.contrib.exporter import BaseItemExporter
-from scrapy.utils.py26 import json
+from scrapy.contrib.exporter import JsonLinesItemExporter
 
-class JsonLinesItemExporter(BaseItemExporter):
-
-    def __init__(self, file, **kwargs):
-        self._configure(kwargs)
-        self.file = file
-        self.encoder = json.JSONEncoder(**kwargs)
-
-    def export_item(self, item):
-        itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(self.encoder.encode(itemdict) + '\n')
+import warnings
+warnings.warn("Module `scrapy.contrib.exporter.jsonlines` is deprecated - use `scrapy.contrib.exporter` instead",
+    DeprecationWarning, stacklevel=2)

scrapy/contrib/pipeline/fileexport.py

@@ -8,7 +8,6 @@ from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core import signals
 from scrapy.core.exceptions import NotConfigured
 from scrapy.contrib import exporter
-from scrapy.contrib.exporter import jsonlines
 from scrapy.conf import settings
 
 class FileExportPipeline(object):
@@ -49,7 +48,9 @@ class FileExportPipeline(object):
         elif format == 'pickle':
             exp = exporter.PickleItemExporter(file, **exp_kwargs)
         elif format == 'json':
-            exp = jsonlines.JsonLinesItemExporter(file, **exp_kwargs)
+            exp = exporter.JsonLinesItemExporter(file, **exp_kwargs)
+        elif format == 'jsonlines':
+            exp = exporter.JsonItemExporter(file, **exp_kwargs)
         else:
             raise NotConfigured("Unsupported export format: %s" % format)
         return exp, file

scrapy/tests/test_contrib_exporter.py

@@ -4,9 +4,9 @@ from cStringIO import StringIO
 from scrapy.item import Item, Field
 from scrapy.utils.python import str_to_unicode
 from scrapy.utils.py26 import json
-from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter
 from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
-    PickleItemExporter, CsvItemExporter, XmlItemExporter
+    PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \
+    JsonItemExporter
 
 class TestItem(Item):
     name = Field()
@@ -156,6 +156,22 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         exported = json.loads(self.output.getvalue().strip())
         self.assertEqual(exported, dict(self.i))
 
+class JsonItemExporterTest(JsonLinesItemExporterTest):
+
+    def _get_exporter(self, **kwargs):
+        return JsonItemExporter(self.output, **kwargs)
+
+    def _check_output(self):
+        exported = json.loads(self.output.getvalue().strip())
+        self.assertEqual(exported, [dict(self.i)])
+
+    def test_two_items(self):
+        self.ie.start_exporting()
+        self.ie.export_item(self.i)
+        self.ie.export_item(self.i)
+        self.ie.finish_exporting()
+        exported = json.loads(self.output.getvalue())
+        self.assertEqual(exported, [dict(self.i), dict(self.i)])
 
 class CustomItemExporterTest(unittest.TestCase):