RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FEED_EXPORT_FIELDS option

This commit is contained in:
Mikhail Korobov 2015-04-15 02:48:26 +05:00
parent 973c31f735
commit 1534e8540b
5 changed files with 165 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
docs/topics/feed-exports.rst
View File

@ -30,7 +30,7 @@ For serializing the scraped data, the feed exports use the :ref:`Item exporters
But you can also extend the supported format through the
:setting:`FEED_EXPORTERS` setting.
.. _topics-feed-format-json:
JSON
@ -38,7 +38,8 @@ JSON
* :setting:`FEED_FORMAT`: ``json``
* Exporter used: :class:`~scrapy.contrib.exporter.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with large feeds
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
.. _topics-feed-format-jsonlines:
@ -55,6 +56,10 @@ CSV
* :setting:`FEED_FORMAT`: ``csv``
* Exporter used: :class:`~scrapy.contrib.exporter.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
.. _topics-feed-format-xml:
@ -202,6 +207,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORAGES`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
.. currentmodule:: scrapy.contrib.feedexport
@ -225,6 +231,20 @@ FEED_FORMAT
The serialization format to be used for the feed. See
:ref:`topics-feed-format` for possible values.
.. setting:: FEED_EXPORT_FIELDS
FEED_EXPORT_FIELDS
------------------
A list of fields to export, optional.
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When omitted, Scrapy uses fields defined in :class:`~.Item` subclasses a spider
is yielding. If raw dicts are used as items Scrapy tries to infer field names
from the exported data - currently it uses field names from the first item.
.. setting:: FEED_STORE_EMPTY
FEED_STORE_EMPTY
@ -249,7 +269,7 @@ The keys are URI schemes and the values are paths to storage classes.
FEED_STORAGES_BASE
------------------
Default::
Default::
{
'': 'scrapy.contrib.feedexport.FileFeedStorage',
@ -277,7 +297,7 @@ classes.
FEED_EXPORTERS_BASE
-------------------
Default::
Default::
FEED_EXPORTERS_BASE = {
'json': 'scrapy.contrib.exporter.JsonItemExporter',

7
scrapy/contrib/feedexport.py
View File

@ -146,6 +146,7 @@ class FeedExporter(object):
if not self._exporter_supported(self.format):
raise NotConfigured
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
uripar = settings['FEED_URI_PARAMS']
self._uripar = load_object(uripar) if uripar else lambda x, y: None
@ -169,7 +170,7 @@ class FeedExporter(object):
uri = self.urifmt % self._get_uri_params(spider)
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(file)
exporter = self._get_exporter(file, fields_to_export=self.export_fields)
exporter.start_exporting()
self.slot = SpiderSlot(file, exporter, storage, uri)
@ -218,8 +219,8 @@ class FeedExporter(object):
else:
log.msg("Unknown feed storage scheme: %s" % scheme, log.ERROR)
def _get_exporter(self, *a, **kw):
return self.exporters[self.format](*a, **kw)
def _get_exporter(self, *args, **kwargs):
return self.exporters[self.format](*args, **kwargs)
def _get_storage(self, uri):
return self.storages[urlparse(uri).scheme](uri)

1
scrapy/settings/default_settings.py
View File

@ -126,6 +126,7 @@ FEED_URI = None
FEED_URI_PARAMS = None # a function to extend uri arguments
FEED_FORMAT = 'jsonlines'
FEED_STORE_EMPTY = False
FEED_EXPORT_FIELDS = None
FEED_STORAGES = {}
FEED_STORAGES_BASE = {
'': 'scrapy.contrib.feedexport.FileFeedStorage',

1
tests/mockserver.py
View File

@ -191,6 +191,7 @@ class MockServer():
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
stdout=PIPE, env=get_testenv())
self.proc.stdout.readline()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()

141
tests/test_contrib_feedexport.py
View File

@ -1,16 +1,27 @@
from __future__ import absolute_import
import os
import csv
from io import BytesIO
import tempfile
import shutil
from six.moves.urllib.parse import urlparse
from zope.interface.verify import verifyObject
from twisted.trial import unittest
from twisted.internet import defer
from scrapy.crawler import CrawlerRunner
from scrapy.settings import Settings
from tests.mockserver import MockServer
from w3lib.url import path_to_file_uri
from scrapy.spider import Spider
from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage
import scrapy
from scrapy.contrib.feedexport import (
IFeedStorage, FileFeedStorage, FTPFeedStorage,
S3FeedStorage, StdoutFeedStorage
)
from scrapy.utils.test import assert_aws_environ
class FileFeedStorageTest(unittest.TestCase):
def test_store_file_uri(self):
@ -39,7 +50,7 @@ class FileFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def _assert_stores(self, storage, path):
spider = Spider("default")
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
yield storage.store(file)
@ -61,7 +72,7 @@ class FTPFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def _assert_stores(self, storage, path):
spider = Spider("default")
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
yield storage.store(file)
@ -85,7 +96,7 @@ class S3FeedStorageTest(unittest.TestCase):
from boto import connect_s3
storage = S3FeedStorage(uri)
verifyObject(IFeedStorage, storage)
file = storage.open(Spider("default"))
file = storage.open(scrapy.Spider("default"))
file.write("content")
yield storage.store(file)
u = urlparse(uri)
@ -99,7 +110,125 @@ class StdoutFeedStorageTest(unittest.TestCase):
def test_store(self):
out = BytesIO()
storage = StdoutFeedStorage('stdout:', _stdout=out)
file = storage.open(Spider("default"))
file = storage.open(scrapy.Spider("default"))
file.write(b"content")
yield storage.store(file)
self.assertEqual(out.getvalue(), b"content")
class FeedExportTest(unittest.TestCase):
class MyItem(scrapy.Item):
foo = scrapy.Field()
egg = scrapy.Field()
baz = scrapy.Field()
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings=None):
""" Run spider with specified settings; return exported data. """
tmpdir = tempfile.mkdtemp()
res_name = tmpdir + '/res'
defaults = {
'FEED_URI': 'file://' + res_name,
'FEED_FORMAT': 'csv',
}
defaults.update(settings or {})
try:
with MockServer() as s:
runner = CrawlerRunner(Settings(defaults))
yield runner.crawl(spider_cls)
with open(res_name, 'rb') as f:
defer.returnValue(f.read())
finally:
shutil.rmtree(tmpdir)
@defer.inlineCallbacks
def exported_data(self, items, settings):
"""
Return exported data which a spider yielding ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = 'testspider'
start_urls = ['http://localhost:8998/']
def parse(self, response):
for item in items:
yield item
data = yield self.run_and_export(TestSpider, settings)
defer.returnValue(data)
@defer.inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
settings = settings or {}
settings.update({'FEED_FORMAT': 'csv'})
data = yield self.exported_data(items, settings)
reader = csv.DictReader(data.splitlines())
got_rows = list(reader)
if ordered:
self.assertEqual(reader.fieldnames, header)
else:
self.assertEqual(set(reader.fieldnames), set(header))
self.assertEqual(rows, got_rows)
@defer.inlineCallbacks
def test_export_csv_items(self):
# feed exporters use field names from Item
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
]
rows = [
{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
header = self.MyItem.fields.keys()
yield self.assertExportedCsv(items, header, rows, ordered=False)
@defer.inlineCallbacks
def test_export_csv_dicts(self):
# When dicts are used, only keys from the first row are used as
# a header.
items = [
{'foo': 'bar', 'egg': 'spam'},
{'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
]
rows = [
{'egg': 'spam', 'foo': 'bar'},
{'egg': 'spam', 'foo': 'bar'}
]
yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False)
@defer.inlineCallbacks
def test_export_csv_feed_export_fields(self):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
for item_cls in [self.MyItem, dict]:
items = [
item_cls({'foo': 'bar1', 'egg': 'spam1'}),
item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
]
# export all columns
settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'}
rows = [
{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows,
settings=settings, ordered=True)
# export a subset of columns
settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
rows = [
{'egg': 'spam1', 'baz': ''},
{'egg': 'spam2', 'baz': 'quux2'}
]
yield self.assertExportedCsv(items, ['egg', 'baz'], rows,
settings=settings, ordered=True)