diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index a97613e6e..1f4a15710 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -64,6 +64,14 @@ XML * :setting:`FEED_FORMAT`: ``xml`` * Exporter used: :class:`~scrapy.contrib.exporter.XmlItemExporter` +.. _topics-feed-format-pickle: + +Pickle +------ + + * :setting:`FEED_FORMAT`: ``pickle`` + * Exporter used: :class:`~scrapy.contrib.exporter.PickleItemExporter` + .. _topics-feed-format-marshal: Marshal diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 820bf06e2..30b164dbb 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -5,7 +5,7 @@ Item Exporters are used to export/serialize items into different formats. import csv import pprint import marshal -from cPickle import Pickler +import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.py26 import json @@ -178,12 +178,14 @@ class CsvItemExporter(BaseItemExporter): class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=0, **kwargs): + def __init__(self, file, protocol=2, **kwargs): self._configure(kwargs) - self.pickler = Pickler(file, protocol) + self.file =file + self.protocol = protocol def export_item(self, item): - self.pickler.dump(dict(self._get_serialized_fields(item))) + d = dict(self._get_serialized_fields(item)) + pickle.dump(d, self.file, self.protocol) class MarshalItemExporter(BaseItemExporter): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c639167a5..7e4363507 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -164,6 +164,7 @@ FEED_EXPORTERS_BASE = { 'csv': 'scrapy.contrib.exporter.CsvItemExporter', 'xml': 'scrapy.contrib.exporter.XmlItemExporter', 'marshal': 'scrapy.contrib.exporter.MarshalItemExporter', + 'pickle': 'scrapy.contrib.exporter.PickleItemExporter', } HTTPCACHE_ENABLED = False diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 974174c42..5039fcf17 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -88,6 +88,19 @@ class PickleItemExporterTest(BaseItemExporterTest): def _check_output(self): self._assert_expected_item(pickle.loads(self.output.getvalue())) + def test_export_multiple_items(self): + i1 = TestItem(name='hello', age='world') + i2 = TestItem(name='bye', age='world') + f = StringIO() + ie = PickleItemExporter(f) + ie.start_exporting() + ie.export_item(i1) + ie.export_item(i2) + ie.finish_exporting() + f.reset() + self.assertEqual(pickle.load(f), i1) + self.assertEqual(pickle.load(f), i2) + class CsvItemExporterTest(BaseItemExporterTest):