From cb95d7a5af416208ba94948ae6d982658f16e85a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 3 Aug 2011 16:16:48 -0300 Subject: [PATCH] added marshal to formats supported by feed exports --- docs/topics/feed-exports.rst | 10 ++++++++++ scrapy/contrib/exporter/__init__.py | 13 ++++++++++++- scrapy/settings/default_settings.py | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 02d184e72..a97613e6e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -64,6 +64,15 @@ XML * :setting:`FEED_FORMAT`: ``xml`` * Exporter used: :class:`~scrapy.contrib.exporter.XmlItemExporter` +.. _topics-feed-format-marshal: + +Marshal +------- + + * :setting:`FEED_FORMAT`: ``marshal`` + * Exporter used: :class:`~scrapy.contrib.exporter.MarshalItemExporter` + + .. _topics-feed-storage: Storages @@ -267,6 +276,7 @@ Default:: 'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter', 'csv': 'scrapy.contrib.exporter.CsvItemExporter', 'xml': 'scrapy.contrib.exporter.XmlItemExporter', + 'marshal': 'scrapy.contrib.exporter.MarshalItemExporter', } A dict containing the built-in feed exporters supported by Scrapy. diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 1d695eb9d..820bf06e2 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -4,6 +4,7 @@ Item Exporters are used to export/serialize items into different formats. import csv import pprint +import marshal from cPickle import Pickler from xml.sax.saxutils import XMLGenerator @@ -12,7 +13,7 @@ from scrapy.utils.py26 import json __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \ - 'JsonItemExporter'] + 'JsonItemExporter', 'MarshalItemExporter'] class BaseItemExporter(object): @@ -185,6 +186,16 @@ class PickleItemExporter(BaseItemExporter): self.pickler.dump(dict(self._get_serialized_fields(item))) +class MarshalItemExporter(BaseItemExporter): + + def __init__(self, file, **kwargs): + self._configure(kwargs) + self.file = file + + def export_item(self, item): + marshal.dump(dict(self._get_serialized_fields(item)), self.file) + + class PprintItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index bfeb98aad..c2438e60b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -162,6 +162,7 @@ FEED_EXPORTERS_BASE = { 'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter', 'csv': 'scrapy.contrib.exporter.CsvItemExporter', 'xml': 'scrapy.contrib.exporter.XmlItemExporter', + 'marshal': 'scrapy.contrib.exporter.MarshalItemExporter', } HTTPCACHE_ENABLED = False