mirror of https://github.com/scrapy/scrapy.git
revise/modernize item exporter example in docs
This commit is contained in:
parent
108f8c4fd2
commit
8a7552370d
|
|
@ -36,38 +36,36 @@ to export
|
|||
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
|
||||
the end of the exporting process
|
||||
|
||||
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
|
||||
Exporter to export scraped items to different files, one per spider::
|
||||
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
|
||||
Item Exporters to group scraped items to different files according to the
|
||||
value of one of their fields::
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
|
||||
class XmlExportPipeline(object):
|
||||
class PerYearXmlExportPipeline(object):
|
||||
"""Distribute items across multiple XML files according to their 'year' field"""
|
||||
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
def open_spider(self, spider):
|
||||
self.year_to_exporter = {}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
pipeline = cls()
|
||||
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
|
||||
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
|
||||
return pipeline
|
||||
def close_spider(self, spider):
|
||||
for exporter in self.year_to_exporter.itervalues():
|
||||
exporter.finish_exporting()
|
||||
exporter.file.close()
|
||||
|
||||
def spider_opened(self, spider):
|
||||
file = open('%s_products.xml' % spider.name, 'w+b')
|
||||
self.files[spider] = file
|
||||
self.exporter = XmlItemExporter(file)
|
||||
self.exporter.start_exporting()
|
||||
def _exporter_for_item(self, item):
|
||||
year = item['year']
|
||||
if year not in self.year_to_exporter:
|
||||
f = open('{}.xml'.format(year), 'w+b')
|
||||
exporter = XmlItemExporter(f)
|
||||
exporter.start_exporting()
|
||||
self.year_to_exporter[year] = exporter
|
||||
return self.year_to_exporter[year]
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self.exporter.finish_exporting()
|
||||
file = self.files.pop(spider)
|
||||
file.close()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
self.exporter.export_item(item)
|
||||
return item
|
||||
def process_item(self, item, spider):
|
||||
exporter = self._exporter_for_item(item)
|
||||
exporter.export_item(item)
|
||||
return item
|
||||
|
||||
|
||||
.. _topics-exporters-field-serialization:
|
||||
|
|
|
|||
Loading…
Reference in New Issue