From 8a7552370de2d31fbd3564e47fa92049c8efee6f Mon Sep 17 00:00:00 2001 From: colinmorris Date: Tue, 31 Oct 2017 17:14:53 -0400 Subject: [PATCH] revise/modernize item exporter example in docs --- docs/topics/exporters.rst | 50 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index b6139af92..28f5ad9c2 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -36,38 +36,36 @@ to export 3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal the end of the exporting process -Here you can see an :doc:`Item Pipeline ` which uses an Item -Exporter to export scraped items to different files, one per spider:: +Here you can see an :doc:`Item Pipeline ` which uses multiple +Item Exporters to group scraped items to different files according to the +value of one of their fields:: - from scrapy import signals - from scrapy.exporters import XmlItemExporter + from scrapy.exporters import XmlItemExporter - class XmlExportPipeline(object): + class PerYearXmlExportPipeline(object): + """Distribute items across multiple XML files according to their 'year' field""" - def __init__(self): - self.files = {} + def open_spider(self, spider): + self.year_to_exporter = {} - @classmethod - def from_crawler(cls, crawler): - pipeline = cls() - crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) - crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) - return pipeline + def close_spider(self, spider): + for exporter in self.year_to_exporter.itervalues(): + exporter.finish_exporting() + exporter.file.close() - def spider_opened(self, spider): - file = open('%s_products.xml' % spider.name, 'w+b') - self.files[spider] = file - self.exporter = XmlItemExporter(file) - self.exporter.start_exporting() + def _exporter_for_item(self, item): + year = item['year'] + if year not in self.year_to_exporter: + f = open('{}.xml'.format(year), 'w+b') + exporter = XmlItemExporter(f) + exporter.start_exporting() + self.year_to_exporter[year] = exporter + return self.year_to_exporter[year] - def spider_closed(self, spider): - self.exporter.finish_exporting() - file = self.files.pop(spider) - file.close() - - def process_item(self, item, spider): - self.exporter.export_item(item) - return item + def process_item(self, item, spider): + exporter = self._exporter_for_item(item) + exporter.export_item(item) + return item .. _topics-exporters-field-serialization: