revise/modernize item exporter example in docs

This commit is contained in:
colinmorris 2017-10-31 17:14:53 -04:00
parent 108f8c4fd2
commit 8a7552370d
1 changed files with 24 additions and 26 deletions

View File

@ -36,38 +36,36 @@ to export
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
Exporter to export scraped items to different files, one per spider::
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from scrapy import signals
from scrapy.exporters import XmlItemExporter
from scrapy.exporters import XmlItemExporter
class XmlExportPipeline(object):
class PerYearXmlExportPipeline(object):
"""Distribute items across multiple XML files according to their 'year' field"""
def __init__(self):
self.files = {}
def open_spider(self, spider):
self.year_to_exporter = {}
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def close_spider(self, spider):
for exporter in self.year_to_exporter.itervalues():
exporter.finish_exporting()
exporter.file.close()
def spider_opened(self, spider):
file = open('%s_products.xml' % spider.name, 'w+b')
self.files[spider] = file
self.exporter = XmlItemExporter(file)
self.exporter.start_exporting()
def _exporter_for_item(self, item):
year = item['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'w+b')
exporter = XmlItemExporter(f)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
def spider_closed(self, spider):
self.exporter.finish_exporting()
file = self.files.pop(spider)
file.close()
def process_item(self, item, spider):
self.exporter.export_item(item)
return item
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item
.. _topics-exporters-field-serialization: