From 484bd0d22a11a04ab775ac4f72c75f3ec9050d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 22 Mar 2019 16:59:29 +0100
Subject: [PATCH 01/11] Allow customizing export column names

---
 docs/topics/exporters.rst       | 35 +++++++++++++++-----
 docs/topics/feed-exports.rst    | 17 +++-------
 scrapy/exporters.py             | 32 +++++++++++++++---
 scrapy/extensions/feedexport.py |  3 +-
 scrapy/settings/__init__.py     | 35 +++++++++++++++++++-
 tests/test_exporters.py         | 15 ++++++++-
 tests/test_feedexport.py        | 57 +++++++++++++++++++++++++++++++++
 7 files changed, 165 insertions(+), 29 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f5048d2da..42a93e459 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -190,14 +190,33 @@ BaseItemExporter
 
    .. attribute:: fields_to_export
 
-      A list with the name of the fields that will be exported, or None if you
-      want to export all fields. Defaults to None.
+      Fields to export, their order [1]_ and their output names.
 
-      Some exporters (like :class:`CsvItemExporter`) respect the order of the
-      fields defined in this attribute.
+      Possible values are:
 
-      Some exporters may require fields_to_export list in order to export the
-      data properly when spiders return dicts (not :class:`~Item` instances).
+      -   ``None`` (all fields [2]_, default)
+
+      -   A list of fields::
+
+              ['field1', 'field2']
+
+      -   A dict [3]_ where keys are fields and values are output names::
+
+              {'field1': 'Field 1', 'field2': 'Field 2'}
+
+      .. [1] Not all exporters respect the specified field order.
+      .. [2] If you yield items as dicts (not :class:`Item` instances),
+             exporters that need to know the fields to export beforehand, like
+             :class:`CsvItemExporter`, only export the fields found in the
+             first item.
+      .. [3] Dicts preserve insertion order since `Python 3.7`_
+             (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
+             of Python, use an OrderedDict_ to enforce a specific field order.
+
+             .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
+             .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
+             .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
+             .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
 
    .. attribute:: export_empty_fields
 
@@ -286,8 +305,8 @@ CsvItemExporter
 
    Exports Items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
-   CSV columns and their order. The :attr:`export_empty_fields` attribute has
-   no effect on this exporter.
+   CSV columns, their order and their column names. The
+   :attr:`export_empty_fields` attribute has no effect on this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index cf70b8aca..968cb8884 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -56,7 +56,7 @@ CSV
 
  * :setting:`FEED_FORMAT`: ``csv``
  * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export and their order use
+ * To specify columns to export, their order and their column names, use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
    option, but it is important for CSV because unlike many other export
    formats CSV uses a fixed header.
@@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS
 
 Default: ``None``
 
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
-defined in dicts or :class:`~.Item` subclasses a spider is yielding.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
 
 .. setting:: FEED_EXPORT_INDENT
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 695c74fec..c05acaca5 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,6 +2,7 @@
 Item Exporters are used to export/serialize items into different formats.
 """
 
+from collections import Mapping
 import csv
 import io
 import sys
@@ -64,6 +65,14 @@ class BaseItemExporter(object):
                 field_iter = six.iterkeys(item.fields)
             else:
                 field_iter = six.iterkeys(item)
+        elif isinstance(self.fields_to_export, Mapping):
+            if include_empty:
+                field_iter = self.fields_to_export.items()
+            else:
+                field_iter = (
+                    (x, y) for x, y in self.fields_to_export.items()
+                    if x in item
+                )
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -71,13 +80,22 @@ class BaseItemExporter(object):
                 field_iter = (x for x in self.fields_to_export if x in item)
 
         for field_name in field_iter:
-            if field_name in item:
-                field = {} if isinstance(item, dict) else item.fields[field_name]
-                value = self.serialize_field(field, field_name, item[field_name])
+            if isinstance(field_name, six.string_types):
+                item_field, output_field = field_name, field_name
+            else:
+                item_field, output_field = field_name
+            if item_field in item:
+                if isinstance(item, dict):
+                    field = {}
+                else:
+                    field = item.fields[item_field]
+                value = self.serialize_field(
+                    field, output_field, item[item_field]
+                )
             else:
                 value = default_value
 
-            yield field_name, value
+            yield output_field, value
 
 
 class JsonLinesItemExporter(BaseItemExporter):
@@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter):
                 else:
                     # use fields declared in Item
                     self.fields_to_export = list(item.fields.keys())
-            row = list(self._build_row(self.fields_to_export))
+            if isinstance(self.fields_to_export, Mapping):
+                fields = self.fields_to_export.values()
+            else:
+                fields = self.fields_to_export
+            row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b2f7267a2..1ed476d83 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -201,7 +201,8 @@ class FeedExporter(object):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
         self._exporting = False
-        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
+        self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS')
+        self.export_fields = self.export_fields or None
         self.indent = None
         if settings.get('FEED_EXPORT_INDENT') is not None:
             self.indent = settings.getint('FEED_EXPORT_INDENT')
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 14c93bef2..69b324e84 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,7 +1,7 @@
 import six
 import json
 import copy
-from collections import MutableMapping
+from collections import MutableMapping, OrderedDict
 from importlib import import_module
 from pprint import pformat
 
@@ -198,6 +198,39 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
+    def getdictorlist(self, name, default=None):
+        """Get a setting value as either an ``OrderedDict`` or a list.
+
+        If the setting is already a dict or a list, a copy of it will be
+        returned.
+
+        If it is a string it will be evaluated as JSON, or as a comma-separated
+        list of strings as a fallback.
+
+        For example, settings populated through environment variables will
+        return:
+
+        -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+            ``'{"key1": "value1", "key2": "value2"}'``
+
+        -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
+        value = self.get(name, default)
+        if value is None:
+            return {}
+        if isinstance(value, six.string_types):
+            try:
+                return json.loads(value, object_pairs_hook=OrderedDict)
+            except ValueError:
+                return value.split(',')
+        return copy.deepcopy(value)
+
     def getwithbase(self, name):
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index cd72c661a..1b3dc14a1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,4 +1,7 @@
+# -*- coding:utf-8 -*-
+
 from __future__ import absolute_import
+from collections import OrderedDict
 import re
 import json
 import marshal
@@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase):
         assert isinstance(name, six.text_type)
         self.assertEqual(name, u'John\xa3')
 
+        ie = self._get_exporter(
+            fields_to_export=OrderedDict([('name', u'名稱')])
+        )
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)),
+            [(u'名稱', u'John\xa3')]
+        )
+
     def test_field_custom_serializer(self):
         def custom_serializer(value):
             return str(int(value) + 2)
@@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
 
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
+        self.output = tempfile.TemporaryFile()
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
@@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(csvsplit(first), csvsplit(second), msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+        self.output.seek(0)
+        self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eef0384cf..4f72c0ff4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from collections import OrderedDict
 import os
 import csv
 import json
@@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
 
+        # fields may be defined as a comma-separated list
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExported(items, header, rows,
+                                  settings=settings, ordered=True)
+
+        # fields may also be defined as a JSON array
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExported(items, header, rows,
+                                  settings=settings, ordered=True)
+
+        # custom output field names can be specified
+        header = OrderedDict((
+            ("foo", "Foo"),
+            ("baz", "Baz"),
+            ("hello", "Hello"),
+        ))
+        settings = {'FEED_EXPORT_FIELDS': header}
+        rows = [
+            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
+            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
+            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
+        ]
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings, ordered=True)
+
+        # custom output field names can be specified as a JSON object
+        header = OrderedDict((
+            ("foo", "Foo"),
+            ("baz", "Baz"),
+            ("hello", "Hello"),
+        ))
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        rows = [
+            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
+            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
+            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
+        ]
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings, ordered=True)
+
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as

From 20719bac5cdc4898e9f01fcf4f92aa781a9d0ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Dec 2019 15:09:43 +0100
Subject: [PATCH 02/11] Fix import error

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c1fff4d95..98421be18 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -224,7 +224,7 @@ class BaseSettings(MutableMapping):
         value = self.get(name, default)
         if value is None:
             return {}
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             try:
                 return json.loads(value, object_pairs_hook=OrderedDict)
             except ValueError:

From 5834088e670d93b2a63ad8afb258e687af0a9b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 18 Feb 2020 14:18:15 +0100
Subject: [PATCH 03/11] Apply feedback

---
 scrapy/settings/__init__.py |   5 +-
 tests/test_feedexport.py    | 109 ++++++++++++++++++------------------
 2 files changed, 56 insertions(+), 58 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 98421be18..6f5b1ef97 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -207,8 +207,7 @@ class BaseSettings(MutableMapping):
         If it is a string it will be evaluated as JSON, or as a comma-separated
         list of strings as a fallback.
 
-        For example, settings populated through environment variables will
-        return:
+        For example, settings populated from the command line will return:
 
         -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
             ``'{"key1": "value1", "key2": "value2"}'``
@@ -223,7 +222,7 @@ class BaseSettings(MutableMapping):
         """
         value = self.get(name, default)
         if value is None:
-            return {}
+            return OrderedDict()
         if isinstance(value, str):
             try:
                 return json.loads(value, object_pairs_hook=OrderedDict)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 291c47702..781cdc543 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,6 +5,7 @@ import warnings
 import tempfile
 import shutil
 import string
+import sys
 from collections import OrderedDict
 from io import BytesIO
 from pathlib import Path
@@ -12,6 +13,7 @@ from unittest import mock
 from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
+import pytest
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -590,78 +592,75 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
-        # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+    @defer.inlineCallbacks
+    def test_export_items_empty_field_list(self):
+        # FEED_EXPORT_FIELDS==[] means the same as default None
+        items = [{'foo': 'bar'}]
+        header = ["foo"]
+        rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
-        yield self.assertExportedJsonLines(items, rows_jl, settings)
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedJsonLines(items, rows, settings)
 
-        # it is possible to override fields using FEED_EXPORT_FIELDS
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # fields may be defined as a comma-separated list
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_comma_separated_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # fields may also be defined as a JSON array
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_json_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # custom output field names can be specified
+    @defer.inlineCallbacks
+    def test_export_items_field_names(self):
+        items = [{'foo': 'bar'}]
         header = OrderedDict((
             ("foo", "Foo"),
-            ("baz", "Baz"),
-            ("hello", "Hello"),
         ))
+        rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
-            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
-            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
-            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
-        ]
         yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings, ordered=True)
+                                  settings=settings)
 
-        # custom output field names can be specified as a JSON object
+    @pytest.mark.skipif(sys.version_info < (3, 7),
+                        reason='Only official in Python 3.7+')
+    @defer.inlineCallbacks
+    def test_export_items_dict_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {
+            'baz': 'Baz',
+            'foo': 'Foo',
+        }
+        rows = [{'Baz': '', 'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, ['Baz', 'Foo'], rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_names(self):
+        items = [{'foo': 'bar'}]
         header = OrderedDict((
             ("foo", "Foo"),
-            ("baz", "Baz"),
-            ("hello", "Hello"),
         ))
+        rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
-        rows = [
-            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
-            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
-            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
-            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
-        ]
         yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings, ordered=True)
+                                  settings=settings)
 
     @defer.inlineCallbacks
     def test_export_dicts(self):
@@ -697,7 +696,7 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
             # export a subset of columns
             settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -706,7 +705,7 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['egg', 'baz'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
     @defer.inlineCallbacks
     def test_export_encoding(self):

From 4605c66a80dabd64924e397580224a667cd73ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 May 2020 12:38:51 +0200
Subject: [PATCH 04/11] Fix AttributeError

---
 scrapy/utils/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 376c1f992..6a6d38a5c 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -114,7 +114,7 @@ def get_sources(use_closest=True):
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
-    out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:

From 3e994bda45ac4f95cec0517c4b1be68a38314914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Jun 2022 20:53:14 +0200
Subject: [PATCH 05/11] Update for Python 3.7+

---
 docs/topics/exporters.rst   | 10 +---------
 scrapy/exporters.py         |  2 +-
 scrapy/settings/__init__.py |  9 ++++-----
 tests/test_exporters.py     |  5 ++---
 tests/test_feedexport.py    | 10 +++-------
 5 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 7580011ac..3c36ef002 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -205,7 +205,7 @@ BaseItemExporter
 
               ['field1', 'field2']
 
-      -   A dict [3]_ where keys are fields and values are output names::
+      -   A dict where keys are fields and values are output names::
 
               {'field1': 'Field 1', 'field2': 'Field 2'}
 
@@ -214,14 +214,6 @@ BaseItemExporter
              all their possible fields, exporters that do not support exporting
              a different subset of fields per item will only export the fields 
              found in the first item exported.
-      .. [3] Dicts preserve insertion order since `Python 3.7`_
-             (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
-             of Python, use an OrderedDict_ to enforce a specific field order.
-
-             .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
-             .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
-             .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
-             .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
 
    .. attribute:: export_empty_fields
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index ad12f26d6..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,13 +2,13 @@
 Item Exporters are used to export/serialize items into different formats.
 """
 
-from collections import Mapping
 import csv
 import io
 import marshal
 import pickle
 import pprint
 import warnings
+from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
 from itemadapter import is_item, ItemAdapter
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 2bbe38481..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,6 +1,5 @@
 import json
 import copy
-from collections import OrderedDict
 from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
@@ -199,7 +198,7 @@ class BaseSettings(MutableMapping):
         return dict(value)
 
     def getdictorlist(self, name, default=None):
-        """Get a setting value as either an ``OrderedDict`` or a list.
+        """Get a setting value as either a :class:`dict` or a :class:`list`.
 
         If the setting is already a dict or a list, a copy of it will be
         returned.
@@ -209,7 +208,7 @@ class BaseSettings(MutableMapping):
 
         For example, settings populated from the command line will return:
 
-        -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+        -   ``{'key1': 'value1', 'key2': 'value2'}`` if set to
             ``'{"key1": "value1", "key2": "value2"}'``
 
         -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
@@ -222,10 +221,10 @@ class BaseSettings(MutableMapping):
         """
         value = self.get(name, default)
         if value is None:
-            return OrderedDict()
+            return {}
         if isinstance(value, str):
             try:
-                return json.loads(value, object_pairs_hook=OrderedDict)
+                return json.loads(value)
             except ValueError:
                 return value.split(',')
         return copy.deepcopy(value)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6ba7428f6..096cd3116 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,7 +4,6 @@ import marshal
 import pickle
 import tempfile
 import unittest
-from collections import OrderedDict
 from io import BytesIO
 from datetime import datetime
 from warnings import catch_warnings, filterwarnings
@@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, 'John\xa3')
 
         ie = self._get_exporter(
-            fields_to_export=OrderedDict([('name', u'名稱')])
+            fields_to_export={'name': '名稱'}
         )
         self.assertEqual(
             list(ie._get_serialized_fields(self.i)),
-            [(u'名稱', u'John\xa3')]
+            [('名稱', 'John\xa3')]
         )
 
     def test_field_custom_serializer(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 83aabbdc7..9098e035d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -11,7 +11,7 @@ import sys
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
-from collections import defaultdict, OrderedDict
+from collections import defaultdict
 from contextlib import ExitStack
 from io import BytesIO
 from logging import getLogger
@@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_items_field_names(self):
         items = [{'foo': 'bar'}]
-        header = OrderedDict((
-            ("foo", "Foo"),
-        ))
+        header = {'foo': 'Foo'}
         rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': header}
         yield self.assertExported(items, list(header.values()), rows,
@@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_items_json_field_names(self):
         items = [{'foo': 'bar'}]
-        header = OrderedDict((
-            ("foo", "Foo"),
-        ))
+        header = {'foo': 'Foo'}
         rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
         yield self.assertExported(items, list(header.values()), rows,

From 1b9ed22becf03311ec014dc9b7e0c09ce87b612c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:27:17 +0200
Subject: [PATCH 06/11] Remove Python < 3.7 leftover

---
 tests/test_feedexport.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9098e035d..946c94bd4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1004,8 +1004,6 @@ class FeedExportTest(FeedExportTestBase):
         yield self.assertExported(items, list(header.values()), rows,
                                   settings=settings)
 
-    @pytest.mark.skipif(sys.version_info < (3, 7),
-                        reason='Only official in Python 3.7+')
     @defer.inlineCallbacks
     def test_export_items_dict_field_names(self):
         items = [{'foo': 'bar'}]

From 24f382fa459434cccfa4c0a8884a48d09d75e243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:31:45 +0200
Subject: [PATCH 07/11] test_feedexport: remove ordered=False

---
 tests/test_feedexport.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 946c94bd4..4006b5957 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -657,8 +657,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -719,7 +719,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -730,10 +730,7 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
+        self.assertEqual(reader.fieldnames, header)
 
         self.assertEqual(rows, got_rows)
 
@@ -886,7 +883,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -958,7 +955,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -968,7 +965,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedCsv(items, header, rows)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1146,7 +1143,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2065,7 +2062,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From 3729c6d26698ae6b8a7ef297606a1c7630d82619 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:33:34 +0200
Subject: [PATCH 08/11] Remove unused import and redundant import

---
 tests/test_feedexport.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 4006b5957..8ef221b70 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -21,7 +21,6 @@ from unittest import mock
 from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
-import pytest
 import lxml.etree
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -731,7 +730,6 @@ class FeedExportTest(FeedExportTestBase):
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
         self.assertEqual(reader.fieldnames, header)
-
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -1815,7 +1813,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_filters(self):
-        import sys
         if "PyPy" in sys.version:
             # https://foss.heptapod.net/pypy/pypy/-/issues/3527
             raise unittest.SkipTest("lzma filters doesn't work in PyPy")

From 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:55:45 +0200
Subject: [PATCH 09/11] Revert "test_feedexport: remove ordered=False"

This reverts commit 24f382fa459434cccfa4c0a8884a48d09d75e243.
---
 tests/test_feedexport.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8ef221b70..fe90501fb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None):
-        yield self.assertExportedCsv(items, header, rows, settings)
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -729,7 +729,11 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        self.assertEqual(reader.fieldnames, header)
+        if ordered:
+            self.assertEqual(reader.fieldnames, header)
+        else:
+            self.assertEqual(set(reader.fieldnames), set(header))
+
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -881,7 +885,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows)
+        yield self.assertExported(items, header, rows, ordered=False)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -953,7 +957,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv)
+        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -963,7 +967,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows)
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1141,7 +1145,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2059,7 +2063,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From bc285f393ca8ff33ef715f98ef3367c973d23ab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 09:00:39 +0200
Subject: [PATCH 10/11] Revert "Revert "test_feedexport: remove ordered=False""

This reverts commit 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e.
---
 tests/test_feedexport.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index fe90501fb..8ef221b70 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -729,11 +729,7 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
-
+        self.assertEqual(reader.fieldnames, header)
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -885,7 +881,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -957,7 +953,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -967,7 +963,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedCsv(items, header, rows)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1145,7 +1141,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2063,7 +2059,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From ec5cf3e9cea3c66aca4cf1aad576f33edca3ad1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 09:10:18 +0200
Subject: [PATCH 11/11] test_feedexport: solve ordered comparison issues

---
 tests/test_feedexport.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8ef221b70..ec48f8d4a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -726,11 +726,9 @@ class FeedExportTest(FeedExportTestBase):
             },
         })
         data = yield self.exported_data(items, settings)
-
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
-        got_rows = list(reader)
-        self.assertEqual(reader.fieldnames, header)
-        self.assertEqual(rows, got_rows)
+        self.assertEqual(reader.fieldnames, list(header))
+        self.assertEqual(rows, list(reader))
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -1141,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
+        yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks