diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dc5912fb9..f0d3e660d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -207,6 +207,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGES` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` + * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` .. currentmodule:: scrapy.extensions.feedexport @@ -231,6 +232,20 @@ FEED_FORMAT The serialization format to be used for the feed. See :ref:`topics-feed-format` for possible values. +.. setting:: FEED_EXPORT_ENCODING + +FEED_EXPORT_ENCODING +-------------------- + +Default: ``None`` + +The encoding to be used for the feed. + +If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output, +which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. + +Use ``utf-8`` if you want UTF-8 for JSON too. + .. setting:: FEED_EXPORT_FIELDS FEED_EXPORT_FIELDS diff --git a/scrapy/exporters.py b/scrapy/exporters.py index d507bcf31..c4b1b3476 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -33,9 +33,9 @@ class BaseItemExporter(object): If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses constructors) """ + self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) - self.encoding = options.pop('encoding', 'utf-8') if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -84,11 +84,13 @@ class JsonLinesItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) self.file = file + kwargs.setdefault('ensure_ascii', not self.encoding) self.encoder = ScrapyJSONEncoder(**kwargs) def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n')) + data = self.encoder.encode(itemdict) + '\n' + self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): @@ -96,6 +98,7 @@ class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) self.file = file + kwargs.setdefault('ensure_ascii', not self.encoding) self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True @@ -111,7 +114,8 @@ class JsonItemExporter(BaseItemExporter): else: self.file.write(b',\n') itemdict = dict(self._get_serialized_fields(item)) - self.file.write(to_bytes(self.encoder.encode(itemdict))) + data = self.encoder.encode(itemdict) + self.file.write(to_bytes(data, self.encoding)) class XmlItemExporter(BaseItemExporter): @@ -120,6 +124,8 @@ class XmlItemExporter(BaseItemExporter): self.item_element = kwargs.pop('item_element', 'item') self.root_element = kwargs.pop('root_element', 'items') self._configure(kwargs) + if not self.encoding: + self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) def start_exporting(self): @@ -169,9 +175,16 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): self._configure(kwargs, dont_fail=True) + if not self.encoding: + self.encoding = 'utf-8' self.include_headers_line = include_headers_line - file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True) - self.csv_writer = csv.writer(file, **kwargs) + self.stream = io.TextIOWrapper( + file, + line_buffering=False, + write_through=True, + encoding=self.encoding + ) if six.PY3 else file + self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued @@ -200,7 +213,7 @@ class CsvItemExporter(BaseItemExporter): def _build_row(self, values): for s in values: try: - yield to_native_str(s) + yield to_native_str(s, self.encoding) except TypeError: yield s @@ -263,6 +276,8 @@ class PythonItemExporter(BaseItemExporter): warnings.warn( "PythonItemExporter will drop support for binary export in the future", ScrapyDeprecationWarning) + if not self.encoding: + self.encoding = 'utf-8' def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5d90ef738..c3fc66de5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -162,6 +162,7 @@ class FeedExporter(object): if not self.urifmt: raise NotConfigured self.format = settings['FEED_FORMAT'].lower() + self.export_encoding = settings['FEED_EXPORT_ENCODING'] self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') if not self._storage_supported(self.urifmt): @@ -185,7 +186,8 @@ class FeedExporter(object): uri = self.urifmt % self._get_uri_params(spider) storage = self._get_storage(uri) file = storage.open(spider) - exporter = self._get_exporter(file, fields_to_export=self.export_fields) + exporter = self._get_exporter(file, fields_to_export=self.export_fields, + encoding=self.export_encoding) exporter.start_exporting() self.slot = SpiderSlot(file, exporter, storage, uri) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e563e56aa..5670cbb57 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -139,6 +139,7 @@ FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments FEED_FORMAT = 'jsonlines' FEED_STORE_EMPTY = False +FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None FEED_STORAGES = {} FEED_STORAGES_BASE = { diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 05f19d33e..353b21927 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -371,3 +371,32 @@ class FeedExportTest(unittest.TestCase): ] yield self.assertExported(items, ['egg', 'baz'], rows, settings=settings, ordered=True) + + @defer.inlineCallbacks + def test_export_encoding(self): + items = [dict({'foo': u'Test\xd6'})] + header = ['foo'] + + formats = { + 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), + 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), + 'xml': u'\nTest\xd6'.encode('utf-8'), + 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), + } + + for format in formats: + settings = {'FEED_FORMAT': format} + data = yield self.exported_data(items, settings) + self.assertEqual(formats[format], data) + + formats = { + 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), + 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), + 'xml': u'\nTest\xd6'.encode('latin-1'), + 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), + } + + for format in formats: + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} + data = yield self.exported_data(items, settings) + self.assertEqual(formats[format], data)