From 1534e8540bf083c8d7beb0264cccea5488ee0250 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Apr 2015 02:48:26 +0500 Subject: [PATCH] FEED_EXPORT_FIELDS option --- docs/topics/feed-exports.rst | 28 +++++- scrapy/contrib/feedexport.py | 7 +- scrapy/settings/default_settings.py | 1 + tests/mockserver.py | 1 + tests/test_contrib_feedexport.py | 141 ++++++++++++++++++++++++++-- 5 files changed, 165 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e81db6489..a2bfdad74 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -30,7 +30,7 @@ For serializing the scraped data, the feed exports use the :ref:`Item exporters But you can also extend the supported format through the :setting:`FEED_EXPORTERS` setting. - + .. _topics-feed-format-json: JSON @@ -38,7 +38,8 @@ JSON * :setting:`FEED_FORMAT`: ``json`` * Exporter used: :class:`~scrapy.contrib.exporter.JsonItemExporter` - * See :ref:`this warning ` if you're using JSON with large feeds + * See :ref:`this warning ` if you're using JSON with + large feeds. .. _topics-feed-format-jsonlines: @@ -55,6 +56,10 @@ CSV * :setting:`FEED_FORMAT`: ``csv`` * Exporter used: :class:`~scrapy.contrib.exporter.CsvItemExporter` + * To specify columns to export and their order use + :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this + option, but it is important for CSV because unlike many other export + formats CSV uses a fixed header. .. _topics-feed-format-xml: @@ -202,6 +207,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGES` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` + * :setting:`FEED_EXPORT_FIELDS` .. currentmodule:: scrapy.contrib.feedexport @@ -225,6 +231,20 @@ FEED_FORMAT The serialization format to be used for the feed. See :ref:`topics-feed-format` for possible values. +.. setting:: FEED_EXPORT_FIELDS + +FEED_EXPORT_FIELDS +------------------ + +A list of fields to export, optional. +Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. + +Use FEED_EXPORT_FIELDS option to define fields to export and their order. + +When omitted, Scrapy uses fields defined in :class:`~.Item` subclasses a spider +is yielding. If raw dicts are used as items Scrapy tries to infer field names +from the exported data - currently it uses field names from the first item. + .. setting:: FEED_STORE_EMPTY FEED_STORE_EMPTY @@ -249,7 +269,7 @@ The keys are URI schemes and the values are paths to storage classes. FEED_STORAGES_BASE ------------------ -Default:: +Default:: { '': 'scrapy.contrib.feedexport.FileFeedStorage', @@ -277,7 +297,7 @@ classes. FEED_EXPORTERS_BASE ------------------- -Default:: +Default:: FEED_EXPORTERS_BASE = { 'json': 'scrapy.contrib.exporter.JsonItemExporter', diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index 92664220c..43ebbe321 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -146,6 +146,7 @@ class FeedExporter(object): if not self._exporter_supported(self.format): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') + self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -169,7 +170,7 @@ class FeedExporter(object): uri = self.urifmt % self._get_uri_params(spider) storage = self._get_storage(uri) file = storage.open(spider) - exporter = self._get_exporter(file) + exporter = self._get_exporter(file, fields_to_export=self.export_fields) exporter.start_exporting() self.slot = SpiderSlot(file, exporter, storage, uri) @@ -218,8 +219,8 @@ class FeedExporter(object): else: log.msg("Unknown feed storage scheme: %s" % scheme, log.ERROR) - def _get_exporter(self, *a, **kw): - return self.exporters[self.format](*a, **kw) + def _get_exporter(self, *args, **kwargs): + return self.exporters[self.format](*args, **kwargs) def _get_storage(self, uri): return self.storages[urlparse(uri).scheme](uri) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5b8dc4eaa..57b9d843c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -126,6 +126,7 @@ FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments FEED_FORMAT = 'jsonlines' FEED_STORE_EMPTY = False +FEED_EXPORT_FIELDS = None FEED_STORAGES = {} FEED_STORAGES_BASE = { '': 'scrapy.contrib.feedexport.FileFeedStorage', diff --git a/tests/mockserver.py b/tests/mockserver.py index d2ba92485..3e4f8c0e5 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -191,6 +191,7 @@ class MockServer(): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) self.proc.stdout.readline() + return self def __exit__(self, exc_type, exc_value, traceback): self.proc.kill() diff --git a/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py index 77eb443d5..e09dacf00 100644 --- a/tests/test_contrib_feedexport.py +++ b/tests/test_contrib_feedexport.py @@ -1,16 +1,27 @@ +from __future__ import absolute_import import os +import csv from io import BytesIO +import tempfile +import shutil from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer +from scrapy.crawler import CrawlerRunner +from scrapy.settings import Settings +from tests.mockserver import MockServer from w3lib.url import path_to_file_uri -from scrapy.spider import Spider -from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage +import scrapy +from scrapy.contrib.feedexport import ( + IFeedStorage, FileFeedStorage, FTPFeedStorage, + S3FeedStorage, StdoutFeedStorage +) from scrapy.utils.test import assert_aws_environ + class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): @@ -39,7 +50,7 @@ class FileFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def _assert_stores(self, storage, path): - spider = Spider("default") + spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") yield storage.store(file) @@ -61,7 +72,7 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def _assert_stores(self, storage, path): - spider = Spider("default") + spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") yield storage.store(file) @@ -85,7 +96,7 @@ class S3FeedStorageTest(unittest.TestCase): from boto import connect_s3 storage = S3FeedStorage(uri) verifyObject(IFeedStorage, storage) - file = storage.open(Spider("default")) + file = storage.open(scrapy.Spider("default")) file.write("content") yield storage.store(file) u = urlparse(uri) @@ -99,7 +110,125 @@ class StdoutFeedStorageTest(unittest.TestCase): def test_store(self): out = BytesIO() storage = StdoutFeedStorage('stdout:', _stdout=out) - file = storage.open(Spider("default")) + file = storage.open(scrapy.Spider("default")) file.write(b"content") yield storage.store(file) self.assertEqual(out.getvalue(), b"content") + + +class FeedExportTest(unittest.TestCase): + + class MyItem(scrapy.Item): + foo = scrapy.Field() + egg = scrapy.Field() + baz = scrapy.Field() + + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings=None): + """ Run spider with specified settings; return exported data. """ + tmpdir = tempfile.mkdtemp() + res_name = tmpdir + '/res' + defaults = { + 'FEED_URI': 'file://' + res_name, + 'FEED_FORMAT': 'csv', + } + defaults.update(settings or {}) + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(defaults)) + yield runner.crawl(spider_cls) + + with open(res_name, 'rb') as f: + defer.returnValue(f.read()) + + finally: + shutil.rmtree(tmpdir) + + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + class TestSpider(scrapy.Spider): + name = 'testspider' + start_urls = ['http://localhost:8998/'] + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @defer.inlineCallbacks + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + settings = settings or {} + settings.update({'FEED_FORMAT': 'csv'}) + data = yield self.exported_data(items, settings) + + reader = csv.DictReader(data.splitlines()) + got_rows = list(reader) + if ordered: + self.assertEqual(reader.fieldnames, header) + else: + self.assertEqual(set(reader.fieldnames), set(header)) + + self.assertEqual(rows, got_rows) + + @defer.inlineCallbacks + def test_export_csv_items(self): + # feed exporters use field names from Item + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + ] + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} + ] + header = self.MyItem.fields.keys() + yield self.assertExportedCsv(items, header, rows, ordered=False) + + @defer.inlineCallbacks + def test_export_csv_dicts(self): + # When dicts are used, only keys from the first row are used as + # a header. + items = [ + {'foo': 'bar', 'egg': 'spam'}, + {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'}, + ] + rows = [ + {'egg': 'spam', 'foo': 'bar'}, + {'egg': 'spam', 'foo': 'bar'} + ] + yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False) + + @defer.inlineCallbacks + def test_export_csv_feed_export_fields(self): + # FEED_EXPORT_FIELDS option allows to order export fields + # and to select a subset of fields to export, both for Items and dicts. + + for item_cls in [self.MyItem, dict]: + items = [ + item_cls({'foo': 'bar1', 'egg': 'spam1'}), + item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + ] + + # export all columns + settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'} + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} + ] + yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows, + settings=settings, ordered=True) + + # export a subset of columns + settings = {'FEED_EXPORT_FIELDS': 'egg,baz'} + rows = [ + {'egg': 'spam1', 'baz': ''}, + {'egg': 'spam2', 'baz': 'quux2'} + ] + yield self.assertExportedCsv(items, ['egg', 'baz'], rows, + settings=settings, ordered=True)