From 1276b12c258b42a68ef79c195b5cbd0dee0e52fa Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 22 May 2014 05:38:31 -0300
Subject: [PATCH 01/15] New SettingsAttribute class

---
 scrapy/settings/__init__.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 345c3c133..825e736e0 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -2,6 +2,31 @@ import json
 from . import default_settings
 
 
+class SettingsAttribute(object):
+
+    """Class for storing data related to settings attributes.
+
+    This class is intended for internal usage, you should try Settings class
+    for settings configuration, not this one.
+    """
+
+    def __init__(self, value, priority):
+        self.value = value
+        self.priority = priority
+
+    def set(self, value, priority):
+        """Sets value if priority is higher or equal than current priority."""
+        if priority >= self.priority:
+            self.value = value
+            self.priority = priority
+
+    def __str__(self):
+        return "<SettingsAttribute value={self.value!r} " \
+               "priority={self.priority}>".format(self=self)
+
+    __repr__ = __str__
+
+
 class Settings(object):
 
     def __init__(self, values=None):

From 384efce0a6a43cc5427cb8df4b9c0f6062b73106 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 24 May 2014 00:37:37 -0300
Subject: [PATCH 02/15] Settings priorities dictionary

---
 scrapy/settings/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 825e736e0..b900b1c8d 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -2,6 +2,14 @@ import json
 from . import default_settings
 
 
+SETTINGS_PRIORITIES = {
+    'default': 0,
+    'command': 10,
+    'project': 20,
+    'cmdline': 40,
+}
+
+
 class SettingsAttribute(object):
 
     """Class for storing data related to settings attributes.

From 95ba68683ef3e57bbd93502bb556352028ffba72 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 24 May 2014 00:42:51 -0300
Subject: [PATCH 03/15] New set and setdict method using SettingsAttribute in
 Settings

---
 scrapy/settings/__init__.py | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b900b1c8d..27ffca148 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,4 +1,6 @@
+import six
 import json
+
 from . import default_settings
 
 
@@ -37,14 +39,18 @@ class SettingsAttribute(object):
 
 class Settings(object):
 
-    def __init__(self, values=None):
-        self.values = values.copy() if values else {}
-        self.global_defaults = default_settings
+    def __init__(self, values=None, priority='project'):
+        self.attributes = {}
+        for name, defvalue in iter_default_settings():
+            self.set(name, defvalue, 'default')
+        if values is not None:
+            self.setdict(values, priority)
 
     def __getitem__(self, opt_name):
-        if opt_name in self.values:
-            return self.values[opt_name]
-        return getattr(self.global_defaults, opt_name, None)
+        value = None
+        if opt_name in self.attributes:
+            value = self.attributes[opt_name].value
+        return value
 
     def get(self, name, default=None):
         return self[name] if self[name] is not None else default
@@ -75,12 +81,25 @@ class Settings(object):
         value = self.get(name)
         if value is None:
             return default or {}
-        if isinstance(value, basestring):
+        if isinstance(value, six.string_types):
             value = json.loads(value)
         if isinstance(value, dict):
             return value
         raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value))
 
+    def set(self, name, value, priority='project'):
+        if isinstance(priority, six.string_types):
+            priority = SETTINGS_PRIORITIES[priority]
+        if name not in self.attributes:
+            self.attributes[name] = SettingsAttribute(value, priority)
+        else:
+            self.attributes[name].set(value, priority)
+
+    def setdict(self, values, priority='project'):
+        for name, value in six.iteritems(values):
+            self.set(name, value, priority)
+
+
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):

From 22572834fb80835d82d851bf6fa134d7ac45877b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 26 May 2014 23:27:29 -0300
Subject: [PATCH 04/15] Deprecate CrawlerSettings, as its functionality is
 replicable by Settings class

---
 scrapy/settings/__init__.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 27ffca148..4fed21899 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,6 +1,8 @@
 import six
 import json
 
+from scrapy.utils.deprecate import create_deprecated_class
+
 from . import default_settings
 
 
@@ -103,7 +105,7 @@ class Settings(object):
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):
-        super(CrawlerSettings, self).__init__(**kw)
+        Settings.__init__(self, **kw)
         self.settings_module = settings_module
         self.overrides = {}
         self.defaults = {}
@@ -115,11 +117,15 @@ class CrawlerSettings(Settings):
             return getattr(self.settings_module, opt_name)
         if opt_name in self.defaults:
             return self.defaults[opt_name]
-        return super(CrawlerSettings, self).__getitem__(opt_name)
+        return Settings.__getitem__(self, opt_name)
 
     def __str__(self):
         return "<CrawlerSettings module=%r>" % self.settings_module
 
+CrawlerSettings = create_deprecated_class(
+    'CrawlerSettings', CrawlerSettings,
+    new_class_path='scrapy.settings.Settings')
+
 
 def iter_default_settings():
     """Return the default settings as an iterator of (name, value) tuples"""

From b4b17049e952e4b8e6311d605044243a6f38b36b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 27 May 2014 05:09:12 -0300
Subject: [PATCH 05/15] Settings and SettingsAtribute tests

---
 scrapy/tests/test_settings.py                 |  82 ---------
 scrapy/tests/test_settings/__init__.py        | 160 ++++++++++++++++++
 .../tests/test_settings/default_settings.py   |   2 +
 3 files changed, 162 insertions(+), 82 deletions(-)
 delete mode 100644 scrapy/tests/test_settings.py
 create mode 100644 scrapy/tests/test_settings/__init__.py
 create mode 100644 scrapy/tests/test_settings/default_settings.py

diff --git a/scrapy/tests/test_settings.py b/scrapy/tests/test_settings.py
deleted file mode 100644
index dad52aed9..000000000
--- a/scrapy/tests/test_settings.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import unittest
-
-from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
-from scrapy.spider import Spider
-
-class SettingsTest(unittest.TestCase):
-
-    def test_get(self):
-        settings = Settings({
-            'TEST_ENABLED1': '1',
-            'TEST_ENABLED2': True,
-            'TEST_ENABLED3': 1,
-            'TEST_DISABLED1': '0',
-            'TEST_DISABLED2': False,
-            'TEST_DISABLED3': 0,
-            'TEST_INT1': 123,
-            'TEST_INT2': '123',
-            'TEST_FLOAT1': 123.45,
-            'TEST_FLOAT2': '123.45',
-            'TEST_LIST1': ['one', 'two'],
-            'TEST_LIST2': 'one,two',
-            'TEST_STR': 'value',
-            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
-            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
-        })
-        assert settings.getbool('TEST_ENABLED1') is True
-        assert settings.getbool('TEST_ENABLED2') is True
-        assert settings.getbool('TEST_ENABLED3') is True
-        assert settings.getbool('TEST_ENABLEDx') is False
-        assert settings.getbool('TEST_ENABLEDx', True) is True
-        assert settings.getbool('TEST_DISABLED1') is False
-        assert settings.getbool('TEST_DISABLED2') is False
-        assert settings.getbool('TEST_DISABLED3') is False
-        self.assertEqual(settings.getint('TEST_INT1'), 123)
-        self.assertEqual(settings.getint('TEST_INT2'), 123)
-        self.assertEqual(settings.getint('TEST_INTx'), 0)
-        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
-        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
-        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LISTx'), [])
-        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
-        self.assertEqual(settings['TEST_STR'], 'value')
-        self.assertEqual(settings.get('TEST_STR'), 'value')
-        self.assertEqual(settings['TEST_STRx'], None)
-        self.assertEqual(settings.get('TEST_STRx'), None)
-        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
-        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT3'), {})
-        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
-        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
-
-
-class CrawlerSettingsTest(unittest.TestCase):
-
-    def test_global_defaults(self):
-        crawler = get_crawler()
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 180)
-
-    def test_defaults(self):
-        crawler = get_crawler()
-        crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 99)
-
-    def test_settings_module(self):
-        crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 3)
-
-    def test_overrides(self):
-        crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
-        crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 15)
-
-
-if __name__ == "__main__":
-    unittest.main()
-
diff --git a/scrapy/tests/test_settings/__init__.py b/scrapy/tests/test_settings/__init__.py
new file mode 100644
index 000000000..37cedfc33
--- /dev/null
+++ b/scrapy/tests/test_settings/__init__.py
@@ -0,0 +1,160 @@
+import six
+import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from scrapy.settings import Settings, SettingsAttribute
+from . import default_settings
+
+
+class SettingsAttributeTest(unittest.TestCase):
+
+    def setUp(self):
+        self.attribute = SettingsAttribute('value', 10)
+
+    def test_set_greater_priority(self):
+        self.attribute.set('value2', 20)
+        self.assertEqual(self.attribute.value, 'value2')
+        self.assertEqual(self.attribute.priority, 20)
+
+    def test_set_equal_priority(self):
+        self.attribute.set('value2', 10)
+        self.assertEqual(self.attribute.value, 'value2')
+        self.assertEqual(self.attribute.priority, 10)
+
+    def test_set_less_priority(self):
+        self.attribute.set('value2', 0)
+        self.assertEqual(self.attribute.value, 'value')
+        self.assertEqual(self.attribute.priority, 10)
+
+
+class SettingsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.settings = Settings()
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
+    @mock.patch('scrapy.settings.default_settings', default_settings)
+    def test_initial_defaults(self):
+        settings = Settings()
+        self.assertEqual(len(settings.attributes), 1)
+        self.assertIn('TEST_DEFAULT', settings.attributes)
+
+        attr = settings.attributes['TEST_DEFAULT']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'defvalue')
+        self.assertEqual(attr.priority, 10)
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
+    @mock.patch('scrapy.settings.default_settings', {})
+    def test_initial_values(self):
+        settings = Settings({'TEST_OPTION': 'value'}, 10)
+        self.assertEqual(len(settings.attributes), 1)
+        self.assertIn('TEST_OPTION', settings.attributes)
+
+        attr = settings.attributes['TEST_OPTION']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.priority, 10)
+
+    def test_set_new_attribute(self):
+        self.settings.attributes = {}
+        self.settings.set('TEST_OPTION', 'value', 0)
+        self.assertIn('TEST_OPTION', self.settings.attributes)
+
+        attr = self.settings.attributes['TEST_OPTION']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.priority, 0)
+
+    def test_set_instance_identity_on_update(self):
+        attr = SettingsAttribute('value', 0)
+        self.settings.attributes = {'TEST_OPTION': attr}
+        self.settings.set('TEST_OPTION', 'othervalue', 10)
+
+        self.assertIn('TEST_OPTION', self.settings.attributes)
+        self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
+
+    def test_set_calls_settings_attributes_methods_on_update(self):
+        with mock.patch.object(SettingsAttribute, '__setattr__') as mock_setattr, \
+                mock.patch.object(SettingsAttribute, 'set') as mock_set:
+
+            attr = SettingsAttribute('value', 10)
+            self.settings.attributes = {'TEST_OPTION': attr}
+            mock_set.reset_mock()
+            mock_setattr.reset_mock()
+
+            for priority in (0, 10, 20):
+                self.settings.set('TEST_OPTION', 'othervalue', priority)
+                mock_set.assert_called_once_with('othervalue', priority)
+                self.assertFalse(mock_setattr.called)
+                mock_set.reset_mock()
+                mock_setattr.reset_mock()
+
+    def test_setdict_alias(self):
+        with mock.patch.object(self.settings, 'set') as mock_set:
+            self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
+            self.assertEqual(mock_set.call_count, 2)
+            calls = [mock.call('TEST_1', 'value1', 10),
+                     mock.call('TEST_2', 'value2', 10)]
+            mock_set.assert_has_calls(calls, any_order=True)
+
+    def test_get(self):
+        test_configuration = {
+            'TEST_ENABLED1': '1',
+            'TEST_ENABLED2': True,
+            'TEST_ENABLED3': 1,
+            'TEST_DISABLED1': '0',
+            'TEST_DISABLED2': False,
+            'TEST_DISABLED3': 0,
+            'TEST_INT1': 123,
+            'TEST_INT2': '123',
+            'TEST_FLOAT1': 123.45,
+            'TEST_FLOAT2': '123.45',
+            'TEST_LIST1': ['one', 'two'],
+            'TEST_LIST2': 'one,two',
+            'TEST_STR': 'value',
+            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
+            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
+        }
+        settings = self.settings
+        settings.attributes = {key: SettingsAttribute(value, 0) for key, value
+                               in six.iteritems(test_configuration)}
+
+        self.assertTrue(settings.getbool('TEST_ENABLED1'))
+        self.assertTrue(settings.getbool('TEST_ENABLED2'))
+        self.assertTrue(settings.getbool('TEST_ENABLED3'))
+        self.assertFalse(settings.getbool('TEST_ENABLEDx'))
+        self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
+        self.assertFalse(settings.getbool('TEST_DISABLED1'))
+        self.assertFalse(settings.getbool('TEST_DISABLED2'))
+        self.assertFalse(settings.getbool('TEST_DISABLED3'))
+        self.assertEqual(settings.getint('TEST_INT1'), 123)
+        self.assertEqual(settings.getint('TEST_INT2'), 123)
+        self.assertEqual(settings.getint('TEST_INTx'), 0)
+        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
+        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
+        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
+        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
+        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
+        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
+        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
+        self.assertEqual(settings.getlist('TEST_LISTx'), [])
+        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
+        self.assertEqual(settings['TEST_STR'], 'value')
+        self.assertEqual(settings.get('TEST_STR'), 'value')
+        self.assertEqual(settings['TEST_STRx'], None)
+        self.assertEqual(settings.get('TEST_STRx'), None)
+        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
+        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
+        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
+        self.assertEqual(settings.getdict('TEST_DICT3'), {})
+        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
+        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
+
+
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/scrapy/tests/test_settings/default_settings.py b/scrapy/tests/test_settings/default_settings.py
new file mode 100644
index 000000000..23005d4c6
--- /dev/null
+++ b/scrapy/tests/test_settings/default_settings.py
@@ -0,0 +1,2 @@
+
+TEST_DEFAULT = 'defvalue'

From 00e6470ed0660b48a000d9188a4f1ebb4d9de64c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 30 May 2014 08:34:42 -0300
Subject: [PATCH 06/15] Fix and extend the documentation of the new Settings
 api

---
 docs/topics/api.rst | 83 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 75 insertions(+), 8 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 4f7eb19d2..f0a12dd03 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -103,25 +103,92 @@ how you :ref:`configure the downloader middlewares
         Start the crawler. This calls :meth:`configure` if it hasn't been called yet.
         Returns a deferred that is fired when the crawl is finished.
 
+.. _topics-api-settings:
+
 Settings API
 ============
 
 .. module:: scrapy.settings
    :synopsis: Settings manager
 
-.. class:: Settings()
+.. attribute:: SETTINGS_PRIORITIES
 
-    This object that provides access to Scrapy settings.
+    Dictionary that sets the key name and priority level of the default
+    settings priorities used in Scrapy.
 
-    .. attribute:: overrides
+    Each item defines a settings entry point, giving it a code name for
+    identification and an integer priority. Greater priorities take more
+    precedence over lesser ones when setting and retrieving values in the
+    :class:`~scrapy.settings.Settings` class.
 
-       Global overrides are the ones that take most precedence, and are usually
-       populated by command-line options.
+    .. highlight:: python
 
-       Overrides should be populated *before* configuring the Crawler object
+    ::
+
+        SETTINGS_PRIORITIES = {
+            'default': 0,
+            'command': 10,
+            'project': 20,
+            'cmdline': 40,
+        }
+
+    For a detailed explanation on each settings sources, see:
+    :ref:`topics-settings`.
+
+.. class:: Settings(values={}, priority='project')
+
+    This object stores Scrapy settings for the configuration of internal
+    components, and can be used for any further customization.
+
+    After instantiation of this class, the new object will have the global
+    default settings described on :ref:`topics-settings-ref` already
+    populated.
+
+    Additional values can be passed on initialization with the ``values``
+    argument, and they would take the ``priority`` level.  If the latter
+    argument is a string, the priority name will be looked up in
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a expecific
+    integer should be provided.
+
+    Once the object is created, new settings can be loaded or updated with the
+    :meth:`~scrapy.settings.Settings.set` method, and can be accessed with the
+    square bracket notation of dictionaries, or with the
+    :meth:`~scrapy.settings.Settings.get` method of the instance and its value
+    conversion variants.  When requesting a stored key, the value with the
+    highest priority will be retrieved.
+
+    .. method:: set(name, value, priority='project')
+
+       Store a key/value attribute with a given priority.
+
+       Settings should be populated *before* configuring the Crawler object
        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-       otherwise they won't have any effect. You don't typically need to worry
-       about overrides unless you are implementing your own Scrapy command.
+       otherwise they won't have any effect.
+
+       :param name: the setting name
+       :type name: string
+
+       :param value: the value to associate with the setting
+       :type value: any
+
+       :param priority: the priority of the setting. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
+
+    .. method:: setdict(values, priority='project')
+
+       Store key/value pairs with a given priority.
+
+       This is a helper function that calls
+       :meth:`~scrapy.settings.Settings.set` for every item of ``values``
+       with the provided ``priority``.
+
+       :param values: the settings names and values
+       :type values: dict
+
+       :param priority: the priority of the settings. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
 
     .. method:: get(name, default=None)
 

From 8a3a6236d924f8c9ef0421b8215b6fea89a94e90 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 10 Jun 2014 10:59:48 -0300
Subject: [PATCH 07/15] Settings topic updated

---
 docs/topics/settings.rst | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 44d9128e9..629fac2dd 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -35,22 +35,23 @@ Settings can be populated using different mechanisms, each of which having a
 different precedence. Here is the list of them in decreasing order of
 precedence:
 
- 1. Global overrides (most precedence)
+ 1. Command line options (most precedence)
  2. Project settings module
  3. Default settings per-command
  4. Default global settings (less precedence)
 
+The population of these settings sources is taken care of internally, but a
+manual handling is possible using API calls. See the
+:ref:`topics-api-settings` topic for reference.
+
 These mechanisms are described in more detail below.
 
-1. Global overrides
--------------------
+1. Command line options
+-----------------------
 
-Global overrides are the ones that take most precedence, and are usually
-populated by command-line options. You can also override one (or more) settings
-from command line using the ``-s`` (or ``--set``) command line option.
-
-For more information see the :attr:`~scrapy.settings.Settings.overrides`
-Settings attribute.
+Arguments provided by the command line are the ones that take most precedence,
+overriding any other options. You can explicitly override one (or more)
+settings using the ``-s`` (or ``--set``) command line option.
 
 .. highlight:: sh
 

From bdca06240c2cb26c844228c0fa914a7cf2cc142a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 9 Jun 2014 18:48:34 -0300
Subject: [PATCH 08/15] Fix settings repr on the logs of the shell and tutorial
 docs topics

---
 docs/intro/tutorial.rst | 2 +-
 docs/topics/shell.rst   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 64b9dca62..1df7fe366 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -262,7 +262,7 @@ This is what the shell looks like::
     [s]   item       {}
     [s]   request    <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
     [s]   response   <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x3fadc50>
     [s]   spider     <Spider 'default' at 0x3cebf50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 8827f7fb6..f0c14dc76 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -118,7 +118,7 @@ all start with the ``[s]`` prefix)::
     [s]   request    <GET http://scrapy.org>
     [s]   response   <200 http://scrapy.org>
     [s]   sel        <Selector xpath=None data=u'<html>\n  <head>\n    <meta charset="utf-8'>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
@@ -139,7 +139,7 @@ After that, we can star playing with the objects::
     [s]   request    <GET http://slashdot.org>
     [s]   response   <200 http://slashdot.org>
     [s]   sel        <Selector xpath=None data=u'<html lang="en">\n<head>\n\n\n\n\n<script id="'>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)

From ed033854e24e66488cc2fd149345376874a12925 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 3 Jun 2014 01:31:34 -0300
Subject: [PATCH 09/15] setmodule helper method on Settings class

---
 docs/topics/api.rst                    | 15 +++++++++++
 scrapy/settings/__init__.py            | 11 ++++++--
 scrapy/tests/test_settings/__init__.py | 35 ++++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index f0a12dd03..341340c2a 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -190,6 +190,21 @@ Settings API
            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
        :type priority: string or int
 
+    .. method:: setmodule(module, priority='project')
+
+       Store settings from a module with a given priority.
+
+       This is a helper function that calls
+       :meth:`~scrapy.settings.Settings.set` for every globally declared
+       uppercase variable of ``module`` with the provided ``priority``.
+
+       :param module: the module or the path of the module
+       :type module: module object or string
+
+       :param priority: the priority of the settings. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
+
     .. method:: get(name, default=None)
 
        Get a setting value without affecting its original type.
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 4fed21899..60af208d6 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,5 +1,6 @@
 import six
 import json
+from importlib import import_module
 
 from scrapy.utils.deprecate import create_deprecated_class
 
@@ -43,8 +44,7 @@ class Settings(object):
 
     def __init__(self, values=None, priority='project'):
         self.attributes = {}
-        for name, defvalue in iter_default_settings():
-            self.set(name, defvalue, 'default')
+        self.setmodule(default_settings, priority='default')
         if values is not None:
             self.setdict(values, priority)
 
@@ -101,6 +101,13 @@ class Settings(object):
         for name, value in six.iteritems(values):
             self.set(name, value, priority)
 
+    def setmodule(self, module, priority='project'):
+        if isinstance(module, six.string_types):
+            module = import_module(module)
+        for key in dir(module):
+            if key.isupper():
+                self.set(key, getattr(module, key), priority)
+
 
 class CrawlerSettings(Settings):
 
diff --git a/scrapy/tests/test_settings/__init__.py b/scrapy/tests/test_settings/__init__.py
index 37cedfc33..34521a3eb 100644
--- a/scrapy/tests/test_settings/__init__.py
+++ b/scrapy/tests/test_settings/__init__.py
@@ -101,6 +101,41 @@ class SettingsTest(unittest.TestCase):
                      mock.call('TEST_2', 'value2', 10)]
             mock_set.assert_has_calls(calls, any_order=True)
 
+    def test_setmodule_only_load_uppercase_vars(self):
+        class ModuleMock():
+            UPPERCASE_VAR = 'value'
+            MIXEDcase_VAR = 'othervalue'
+            lowercase_var = 'anothervalue'
+
+        self.settings.attributes = {}
+        self.settings.setmodule(ModuleMock(), 10)
+        self.assertIn('UPPERCASE_VAR', self.settings.attributes)
+        self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
+        self.assertNotIn('lowercase_var', self.settings.attributes)
+        self.assertEqual(len(self.settings.attributes), 1)
+
+    def test_setmodule_alias(self):
+        with mock.patch.object(self.settings, 'set') as mock_set:
+            self.settings.setmodule(default_settings, 10)
+            mock_set.assert_called_with('TEST_DEFAULT', 'defvalue', 10)
+
+    def test_setmodule_by_path(self):
+        self.settings.attributes = {}
+        self.settings.setmodule(default_settings, 10)
+        ctrl_attributes = self.settings.attributes.copy()
+
+        self.settings.attributes = {}
+        self.settings.setmodule(
+            'scrapy.tests.test_settings.default_settings', 10)
+
+        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+                              six.iterkeys(ctrl_attributes))
+
+        for attr, ctrl_attr in zip(six.itervalues(self.settings.attributes),
+                                   six.itervalues(ctrl_attributes)):
+            self.assertEqual(attr.value, ctrl_attr.value)
+            self.assertEqual(attr.priority, ctrl_attr.priority)
+
     def test_get(self):
         test_configuration = {
             'TEST_ENABLED1': '1',

From b397f5e935c616b1d4887ef8654f185321d35731 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 24 May 2014 08:06:00 -0300
Subject: [PATCH 10/15] Update get_crawler method in utils/test.py with new
 Settings interface

This early adaptation of the new Settings interface is needed for running tests
---
 scrapy/utils/test.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 8c1f7fe8a..e6376d519 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -22,20 +22,13 @@ def assert_aws_environ():
 
 def get_crawler(settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
-    will be used as the settings present in the settings module of the
-    CrawlerSettings.
+    will be used to populate the crawler settings with a project level
+    priority.
     """
     from scrapy.crawler import Crawler
-    from scrapy.settings import CrawlerSettings
+    from scrapy.settings import Settings
 
-    class SettingsModuleMock(object):
-        pass
-    settings_module = SettingsModuleMock()
-    if settings_dict:
-        for k, v in settings_dict.items():
-            setattr(settings_module, k, v)
-    settings = CrawlerSettings(settings_module)
-    return Crawler(settings)
+    return Crawler(Settings(settings_dict))
 
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this

From c496468e2ac7b87e5d9ad16a3cf4a7f65b10ff5a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 3 Jun 2014 03:43:11 -0300
Subject: [PATCH 11/15] get_project_settings now returns a Settings instance

---
 scrapy/utils/project.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 332fc18b8..34cd8e520 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -6,7 +6,7 @@ from importlib import import_module
 from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
-from scrapy.settings import CrawlerSettings
+from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
@@ -53,20 +53,21 @@ def get_project_settings():
     if ENVVAR not in os.environ:
         project = os.environ.get('SCRAPY_PROJECT', 'default')
         init_env(project)
+
+    settings = Settings()
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
-        settings_module = import_module(settings_module_path)
-    else:
-        settings_module = None
-    settings = CrawlerSettings(settings_module)
+        settings.setmodule(settings_module_path, priority='project')
 
     # XXX: remove this hack
     pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-    settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
+    if pickled_settings:
+        settings.setdict(pickle.loads(pickled_settings), priority='project')
 
     # XXX: deprecate and remove this functionality
-    for k, v in os.environ.items():
-        if k.startswith('SCRAPY_'):
-            settings.overrides[k[7:]] = v
+    env_overrides = {k[7:]: v for k, v in os.environ.items() if
+                     k.startswith('SCRAPY_')}
+    if env_overrides:
+        settings.setdict(env_overrides, priority='project')
 
     return settings

From 080a28f3e978bc184b1c0b94233b013fd0e8d561 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 3 Jun 2014 04:20:13 -0300
Subject: [PATCH 12/15] Change command's default_settings population in
 cmdline.py

---
 scrapy/cmdline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 4601cebfc..6a172cbc6 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -133,7 +133,7 @@ def execute(argv=None, settings=None):
     cmd = cmds[cmdname]
     parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
     parser.description = cmd.long_desc()
-    settings.defaults.update(cmd.default_settings)
+    settings.setdict(cmd.default_settings, priority='command')
     cmd.settings = settings
     cmd.add_options(parser)
     opts, args = parser.parse_args(args=argv[1:])

From c81b087de18d4410554ee398122aff8479d8e84a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 3 Jun 2014 04:21:17 -0300
Subject: [PATCH 13/15] Change how settings are overriden in ScrapyCommands

---
 scrapy/command.py            | 13 +++++++------
 scrapy/commands/crawl.py     |  6 +++---
 scrapy/commands/runspider.py |  6 +++---
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/scrapy/command.py b/scrapy/command.py
index 5fe884abe..b2eb9cf8f 100644
--- a/scrapy/command.py
+++ b/scrapy/command.py
@@ -103,20 +103,21 @@ class ScrapyCommand(object):
 
     def process_options(self, args, opts):
         try:
-            self.settings.overrides.update(arglist_to_dict(opts.set))
+            self.settings.setdict(arglist_to_dict(opts.set),
+                                  priority='cmdline')
         except ValueError:
             raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
-            self.settings.overrides['LOG_ENABLED'] = True
-            self.settings.overrides['LOG_FILE'] = opts.logfile
+            self.settings.set('LOG_ENABLED', True, priority='cmdline')
+            self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
 
         if opts.loglevel:
-            self.settings.overrides['LOG_ENABLED'] = True
-            self.settings.overrides['LOG_LEVEL'] = opts.loglevel
+            self.settings.set('LOG_ENABLED', True, priority='cmdline')
+            self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
 
         if opts.nolog:
-            self.settings.overrides['LOG_ENABLED'] = False
+            self.settings.set('LOG_ENABLED', False, priority='cmdline')
 
         if opts.pidfile:
             with open(opts.pidfile, "w") as f:
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 1cd40469b..5797fd81b 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -31,9 +31,9 @@ class Command(ScrapyCommand):
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
             if opts.output == '-':
-                self.settings.overrides['FEED_URI'] = 'stdout:'
+                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
-                self.settings.overrides['FEED_URI'] = opts.output
+                self.settings.set('FEED_URI', opts.output, priority='cmdline')
             valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
@@ -42,7 +42,7 @@ class Command(ScrapyCommand):
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
                                                                   tuple(valid_output_formats)))
-            self.settings.overrides['FEED_FORMAT'] = opts.output_format
+            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) < 1:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 2d537dee7..89d3337b6 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -54,9 +54,9 @@ class Command(ScrapyCommand):
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
             if opts.output == '-':
-                self.settings.overrides['FEED_URI'] = 'stdout:'
+                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
-                self.settings.overrides['FEED_URI'] = opts.output
+                self.settings.set('FEED_URI', opts.output, priority='cmdline')
             valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
@@ -65,7 +65,7 @@ class Command(ScrapyCommand):
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
                                                                   tuple(valid_output_formats)))
-            self.settings.overrides['FEED_FORMAT'] = opts.output_format
+            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) != 1:

From 43ae69e436992285292aa96b7eabc74df45a206e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 9 Jun 2014 12:20:09 -0300
Subject: [PATCH 14/15] Fix settings usage in runspider and crawl commands

Explicit conversion of FEED_EXPORTERS and FEED_EXPORTERS_BASE values into
dictionaries, and cast of the keys to a list for py3 support.
---
 scrapy/commands/crawl.py     | 5 ++++-
 scrapy/commands/runspider.py | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 5797fd81b..0976de51b 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -34,7 +34,10 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
+            valid_output_formats = (
+                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
+                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
+            )
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 89d3337b6..b1d501842 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -57,7 +57,10 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
+            valid_output_formats = (
+                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
+                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
+            )
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:

From 6730e3c9f73f6864acdea51c88f6c2277b7ae439 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 9 Jun 2014 16:51:25 -0300
Subject: [PATCH 15/15] Fix settings usage across tests

---
 scrapy/tests/test_cmdline/extensions.py             | 2 +-
 scrapy/tests/test_downloadermiddleware_robotstxt.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/tests/test_cmdline/extensions.py b/scrapy/tests/test_cmdline/extensions.py
index a7acf1210..4d347966a 100644
--- a/scrapy/tests/test_cmdline/extensions.py
+++ b/scrapy/tests/test_cmdline/extensions.py
@@ -3,7 +3,7 @@
 class TestExtension(object):
 
     def __init__(self, settings):
-        settings.overrides['TEST1'] = "%s + %s" % (settings['TEST1'], 'started')
+        settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started'))
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/scrapy/tests/test_downloadermiddleware_robotstxt.py
index a98522510..31d7f0087 100644
--- a/scrapy/tests/test_downloadermiddleware_robotstxt.py
+++ b/scrapy/tests/test_downloadermiddleware_robotstxt.py
@@ -6,17 +6,17 @@ from twisted.trial import unittest
 from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
-from scrapy.settings import CrawlerSettings
+from scrapy.settings import Settings
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
 
     def test(self):
         crawler = mock.MagicMock()
-        crawler.settings = CrawlerSettings()
-        crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
+        crawler.settings = Settings()
+        crawler.settings.set('USER_AGENT', 'CustomAgent')
         self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
-        crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
+        crawler.settings.set('ROBOTSTXT_OBEY', True)
         crawler.engine.download = mock.MagicMock()
         ROBOTS = re.sub(r'^\s+(?m)', '', '''
         User-Agent: *