diff --git a/docs/faq.rst b/docs/faq.rst index 309f69f27..57fde2deb 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -160,3 +160,10 @@ Amazon EC2 `_. .. _twistd: http://twistedmatrix.com/documents/current/core/howto/basics.html#auto1 .. _official APT repos: http://dev.scrapy.org/wiki/AptRepos .. _upstart: http://upstart.ubuntu.com/ + +Can I use JSON for large exports? +--------------------------------- + +It'll depend on how large your output is. See :ref:`this warning +` in :class:`~scrapy.contrib.exporter.JsonItemExporter` +documentation. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index cbde79421..8589892ec 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -332,11 +332,38 @@ PprintItemExporter Longer lines (when present) are pretty-formatted. -JsonLinesItemExporter +JsonItemExporter --------------------- -.. module:: scrapy.contrib.exporter.jsonlines - :synopsis: JsonLines Item Exporter +.. class:: JsonItemExporter(file, \**kwargs) + + Exports Items in JSON format to the specified file-like object, writing all + objects as a list of objects. The additional constructor arguments are + passed to the :class:`BaseItemExporter` constructor, and the leftover + arguments to the `JSONEncoder`_ constructor, so you can use any + `JSONEncoder`_ constructor argument to customize this exporter. + + :param file: the file-like object to use for exporting the data. + + A typical output of this exporter would be:: + + [{"name": "Color TV", "price": "1200"}, + {"name": "DVD player", "price": "200"}] + + .. _json-with-large-data: + + .. warning:: JSON is very simple and flexible serialization format, but it + doesn't scale well for large amounts of data since incremental (aka. + stream-mode) parsing is not well supported (if at all) among JSON parsers + (on any language), and most of them just parse the entire object in + memory. If you want the power and simplicity of JSON with a more + stream-friendly format, consider using :class:`JsonLinesItemExporter` + instead, or splitting the output in multiple chunks. + +.. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder + +JsonLinesItemExporter +--------------------- .. class:: JsonLinesItemExporter(file, \**kwargs) @@ -353,4 +380,7 @@ JsonLinesItemExporter {"name": "Color TV", "price": "1200"} {"name": "DVD player", "price": "200"} + Unlike the one produced by :class:`JsonItemExporter`, the format produced by + this exporter is well suited for serializing large amounts of data. + .. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index ba69c93b3..a32250fd8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -178,7 +178,9 @@ on the respective Item Exporter to get more info. the first line. This format requires you to specify the fields to export using the :setting:`EXPORT_FIELDS` setting. -* ``json``: uses a :class:`~jsonlines.JsonLinesItemExporter` +* ``json``: uses a :class:`JsonItemExporter` + +* ``jsonlines``: uses a :class:`JsonLinesItemExporter` * ``pickle``: uses a :class:`PickleItemExporter` diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 90ad47cc5..b77c747cc 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -7,9 +7,12 @@ import pprint from cPickle import Pickler from xml.sax.saxutils import XMLGenerator +from scrapy.utils.py26 import json + __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ - 'CsvItemExporter', 'XmlItemExporter'] + 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \ + 'JsonItemExporter'] class BaseItemExporter(object): @@ -71,6 +74,41 @@ class BaseItemExporter(object): yield field_name, value +class JsonLinesItemExporter(BaseItemExporter): + + def __init__(self, file, **kwargs): + self._configure(kwargs) + self.file = file + self.encoder = json.JSONEncoder(**kwargs) + + def export_item(self, item): + itemdict = dict(self._get_serialized_fields(item)) + self.file.write(self.encoder.encode(itemdict) + '\n') + + +class JsonItemExporter(JsonLinesItemExporter): + + def __init__(self, file, **kwargs): + self._configure(kwargs) + self.file = file + self.encoder = json.JSONEncoder(**kwargs) + self.first_item = True + + def start_exporting(self): + self.file.write("[") + + def finish_exporting(self): + self.file.write("]") + + def export_item(self, item): + if self.first_item: + self.first_item = False + else: + self.file.write(',\n') + itemdict = dict(self._get_serialized_fields(item)) + self.file.write(self.encoder.encode(itemdict)) + + class XmlItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index d26e9607b..eb2dbcafe 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -1,13 +1,5 @@ -from scrapy.contrib.exporter import BaseItemExporter -from scrapy.utils.py26 import json +from scrapy.contrib.exporter import JsonLinesItemExporter -class JsonLinesItemExporter(BaseItemExporter): - - def __init__(self, file, **kwargs): - self._configure(kwargs) - self.file = file - self.encoder = json.JSONEncoder(**kwargs) - - def export_item(self, item): - itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict) + '\n') +import warnings +warnings.warn("Module `scrapy.contrib.exporter.jsonlines` is deprecated - use `scrapy.contrib.exporter` instead", + DeprecationWarning, stacklevel=2) diff --git a/scrapy/contrib/pipeline/fileexport.py b/scrapy/contrib/pipeline/fileexport.py index 6e2109d01..dcbbcf994 100644 --- a/scrapy/contrib/pipeline/fileexport.py +++ b/scrapy/contrib/pipeline/fileexport.py @@ -8,7 +8,6 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.exceptions import NotConfigured from scrapy.contrib import exporter -from scrapy.contrib.exporter import jsonlines from scrapy.conf import settings class FileExportPipeline(object): @@ -49,7 +48,9 @@ class FileExportPipeline(object): elif format == 'pickle': exp = exporter.PickleItemExporter(file, **exp_kwargs) elif format == 'json': - exp = jsonlines.JsonLinesItemExporter(file, **exp_kwargs) + exp = exporter.JsonLinesItemExporter(file, **exp_kwargs) + elif format == 'jsonlines': + exp = exporter.JsonItemExporter(file, **exp_kwargs) else: raise NotConfigured("Unsupported export format: %s" % format) return exp, file diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 140108b42..8b776f587 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -4,9 +4,9 @@ from cStringIO import StringIO from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode from scrapy.utils.py26 import json -from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ - PickleItemExporter, CsvItemExporter, XmlItemExporter + PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \ + JsonItemExporter class TestItem(Item): name = Field() @@ -156,6 +156,22 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): exported = json.loads(self.output.getvalue().strip()) self.assertEqual(exported, dict(self.i)) +class JsonItemExporterTest(JsonLinesItemExporterTest): + + def _get_exporter(self, **kwargs): + return JsonItemExporter(self.output, **kwargs) + + def _check_output(self): + exported = json.loads(self.output.getvalue().strip()) + self.assertEqual(exported, [dict(self.i)]) + + def test_two_items(self): + self.ie.start_exporting() + self.ie.export_item(self.i) + self.ie.export_item(self.i) + self.ie.finish_exporting() + exported = json.loads(self.output.getvalue()) + self.assertEqual(exported, [dict(self.i), dict(self.i)]) class CustomItemExporterTest(unittest.TestCase):