Merge branch 'settings-cleanup' of https://github.com/Curita/scrapy into Curita-settings-cleanup

docs/intro/tutorial.rst

@@ -262,7 +262,7 @@ This is what the shell looks like::
     [s]   item       {}
     [s]   request    <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
     [s]   response   <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x3fadc50>
     [s]   spider     <Spider 'default' at 0x3cebf50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)

docs/topics/api.rst

@@ -103,25 +103,107 @@ how you :ref:`configure the downloader middlewares
         Start the crawler. This calls :meth:`configure` if it hasn't been called yet.
         Returns a deferred that is fired when the crawl is finished.
 
+.. _topics-api-settings:
+
 Settings API
 ============
 
 .. module:: scrapy.settings
    :synopsis: Settings manager
 
-.. class:: Settings()
+.. attribute:: SETTINGS_PRIORITIES
 
-    This object that provides access to Scrapy settings.
+    Dictionary that sets the key name and priority level of the default
+    settings priorities used in Scrapy.
 
-    .. attribute:: overrides
+    Each item defines a settings entry point, giving it a code name for
+    identification and an integer priority. Greater priorities take more
+    precedence over lesser ones when setting and retrieving values in the
+    :class:`~scrapy.settings.Settings` class.
 
-       Global overrides are the ones that take most precedence, and are usually
-       populated by command-line options.
+    .. highlight:: python
 
-       Overrides should be populated *before* configuring the Crawler object
+    ::
+
+        SETTINGS_PRIORITIES = {
+            'default': 0,
+            'command': 10,
+            'project': 20,
+            'cmdline': 40,
+        }
+
+    For a detailed explanation on each settings sources, see:
+    :ref:`topics-settings`.
+
+.. class:: Settings(values={}, priority='project')
+
+    This object stores Scrapy settings for the configuration of internal
+    components, and can be used for any further customization.
+
+    After instantiation of this class, the new object will have the global
+    default settings described on :ref:`topics-settings-ref` already
+    populated.
+
+    Additional values can be passed on initialization with the ``values``
+    argument, and they would take the ``priority`` level.  If the latter
+    argument is a string, the priority name will be looked up in
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a expecific
+    integer should be provided.
+
+    Once the object is created, new settings can be loaded or updated with the
+    :meth:`~scrapy.settings.Settings.set` method, and can be accessed with the
+    square bracket notation of dictionaries, or with the
+    :meth:`~scrapy.settings.Settings.get` method of the instance and its value
+    conversion variants.  When requesting a stored key, the value with the
+    highest priority will be retrieved.
+
+    .. method:: set(name, value, priority='project')
+
+       Store a key/value attribute with a given priority.
+
+       Settings should be populated *before* configuring the Crawler object
        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-       otherwise they won't have any effect. You don't typically need to worry
-       about overrides unless you are implementing your own Scrapy command.
+       otherwise they won't have any effect.
+
+       :param name: the setting name
+       :type name: string
+
+       :param value: the value to associate with the setting
+       :type value: any
+
+       :param priority: the priority of the setting. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
+
+    .. method:: setdict(values, priority='project')
+
+       Store key/value pairs with a given priority.
+
+       This is a helper function that calls
+       :meth:`~scrapy.settings.Settings.set` for every item of ``values``
+       with the provided ``priority``.
+
+       :param values: the settings names and values
+       :type values: dict
+
+       :param priority: the priority of the settings. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
+
+    .. method:: setmodule(module, priority='project')
+
+       Store settings from a module with a given priority.
+
+       This is a helper function that calls
+       :meth:`~scrapy.settings.Settings.set` for every globally declared
+       uppercase variable of ``module`` with the provided ``priority``.
+
+       :param module: the module or the path of the module
+       :type module: module object or string
+
+       :param priority: the priority of the settings. Should be a key of
+           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+       :type priority: string or int
 
     .. method:: get(name, default=None)
 

docs/topics/settings.rst

@@ -35,22 +35,23 @@ Settings can be populated using different mechanisms, each of which having a
 different precedence. Here is the list of them in decreasing order of
 precedence:
 
- 1. Global overrides (most precedence)
+ 1. Command line options (most precedence)
  2. Project settings module
  3. Default settings per-command
  4. Default global settings (less precedence)
 
+The population of these settings sources is taken care of internally, but a
+manual handling is possible using API calls. See the
+:ref:`topics-api-settings` topic for reference.
+
 These mechanisms are described in more detail below.
 
-1. Global overrides
--------------------
+1. Command line options
+-----------------------
 
-Global overrides are the ones that take most precedence, and are usually
-populated by command-line options. You can also override one (or more) settings
-from command line using the ``-s`` (or ``--set``) command line option.
-
-For more information see the :attr:`~scrapy.settings.Settings.overrides`
-Settings attribute.
+Arguments provided by the command line are the ones that take most precedence,
+overriding any other options. You can explicitly override one (or more)
+settings using the ``-s`` (or ``--set``) command line option.
 
 .. highlight:: sh
 

docs/topics/shell.rst

@@ -118,7 +118,7 @@ all start with the ``[s]`` prefix)::
     [s]   request    <GET http://scrapy.org>
     [s]   response   <200 http://scrapy.org>
     [s]   sel        <Selector xpath=None data=u'<html>\n  <head>\n    <meta charset="utf-8'>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
@@ -139,7 +139,7 @@ After that, we can star playing with the objects::
     [s]   request    <GET http://slashdot.org>
     [s]   response   <200 http://slashdot.org>
     [s]   sel        <Selector xpath=None data=u'<html lang="en">\n<head>\n\n\n\n\n<script id="'>
-    [s]   settings   <CrawlerSettings module=None>
+    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)

scrapy/cmdline.py

@@ -133,7 +133,7 @@ def execute(argv=None, settings=None):
     cmd = cmds[cmdname]
     parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
     parser.description = cmd.long_desc()
-    settings.defaults.update(cmd.default_settings)
+    settings.setdict(cmd.default_settings, priority='command')
     cmd.settings = settings
     cmd.add_options(parser)
     opts, args = parser.parse_args(args=argv[1:])

scrapy/command.py

@@ -103,20 +103,21 @@ class ScrapyCommand(object):
 
     def process_options(self, args, opts):
         try:
-            self.settings.overrides.update(arglist_to_dict(opts.set))
+            self.settings.setdict(arglist_to_dict(opts.set),
+                                  priority='cmdline')
         except ValueError:
             raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
-            self.settings.overrides['LOG_ENABLED'] = True
-            self.settings.overrides['LOG_FILE'] = opts.logfile
+            self.settings.set('LOG_ENABLED', True, priority='cmdline')
+            self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
 
         if opts.loglevel:
-            self.settings.overrides['LOG_ENABLED'] = True
-            self.settings.overrides['LOG_LEVEL'] = opts.loglevel
+            self.settings.set('LOG_ENABLED', True, priority='cmdline')
+            self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
 
         if opts.nolog:
-            self.settings.overrides['LOG_ENABLED'] = False
+            self.settings.set('LOG_ENABLED', False, priority='cmdline')
 
         if opts.pidfile:
             with open(opts.pidfile, "w") as f:

scrapy/commands/crawl.py

@@ -31,10 +31,13 @@ class Command(ScrapyCommand):
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
             if opts.output == '-':
-                self.settings.overrides['FEED_URI'] = 'stdout:'
+                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
-                self.settings.overrides['FEED_URI'] = opts.output
-            valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
+                self.settings.set('FEED_URI', opts.output, priority='cmdline')
+            valid_output_formats = (
+                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
+                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
+            )
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:
@@ -42,7 +45,7 @@ class Command(ScrapyCommand):
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
                                                                   tuple(valid_output_formats)))
-            self.settings.overrides['FEED_FORMAT'] = opts.output_format
+            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) < 1:

scrapy/commands/runspider.py

@@ -54,10 +54,13 @@ class Command(ScrapyCommand):
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
             if opts.output == '-':
-                self.settings.overrides['FEED_URI'] = 'stdout:'
+                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
-                self.settings.overrides['FEED_URI'] = opts.output
-            valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
+                self.settings.set('FEED_URI', opts.output, priority='cmdline')
+            valid_output_formats = (
+                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
+                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
+            )
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:
@@ -65,7 +68,7 @@ class Command(ScrapyCommand):
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
                                                                   tuple(valid_output_formats)))
-            self.settings.overrides['FEED_FORMAT'] = opts.output_format
+            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) != 1:

scrapy/settings/__init__.py

@@ -1,17 +1,58 @@
+import six
 import json
+from importlib import import_module
+
+from scrapy.utils.deprecate import create_deprecated_class
+
 from . import default_settings
 
 
+SETTINGS_PRIORITIES = {
+    'default': 0,
+    'command': 10,
+    'project': 20,
+    'cmdline': 40,
+}
+
+
+class SettingsAttribute(object):
+
+    """Class for storing data related to settings attributes.
+
+    This class is intended for internal usage, you should try Settings class
+    for settings configuration, not this one.
+    """
+
+    def __init__(self, value, priority):
+        self.value = value
+        self.priority = priority
+
+    def set(self, value, priority):
+        """Sets value if priority is higher or equal than current priority."""
+        if priority >= self.priority:
+            self.value = value
+            self.priority = priority
+
+    def __str__(self):
+        return "<SettingsAttribute value={self.value!r} " \
+               "priority={self.priority}>".format(self=self)
+
+    __repr__ = __str__
+
+
 class Settings(object):
 
-    def __init__(self, values=None):
-        self.values = values.copy() if values else {}
-        self.global_defaults = default_settings
+    def __init__(self, values=None, priority='project'):
+        self.attributes = {}
+        self.setmodule(default_settings, priority='default')
+        if values is not None:
+            self.setdict(values, priority)
 
     def __getitem__(self, opt_name):
-        if opt_name in self.values:
-            return self.values[opt_name]
-        return getattr(self.global_defaults, opt_name, None)
+        value = None
+        if opt_name in self.attributes:
+            value = self.attributes[opt_name].value
+        return value
 
     def get(self, name, default=None):
         return self[name] if self[name] is not None else default
@@ -42,16 +83,36 @@ class Settings(object):
         value = self.get(name)
         if value is None:
             return default or {}
-        if isinstance(value, basestring):
+        if isinstance(value, six.string_types):
             value = json.loads(value)
         if isinstance(value, dict):
             return value
         raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value))
 
+    def set(self, name, value, priority='project'):
+        if isinstance(priority, six.string_types):
+            priority = SETTINGS_PRIORITIES[priority]
+        if name not in self.attributes:
+            self.attributes[name] = SettingsAttribute(value, priority)
+        else:
+            self.attributes[name].set(value, priority)
+
+    def setdict(self, values, priority='project'):
+        for name, value in six.iteritems(values):
+            self.set(name, value, priority)
+
+    def setmodule(self, module, priority='project'):
+        if isinstance(module, six.string_types):
+            module = import_module(module)
+        for key in dir(module):
+            if key.isupper():
+                self.set(key, getattr(module, key), priority)
+
+
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):
-        super(CrawlerSettings, self).__init__(**kw)
+        Settings.__init__(self, **kw)
         self.settings_module = settings_module
         self.overrides = {}
         self.defaults = {}
@@ -63,11 +124,15 @@ class CrawlerSettings(Settings):
             return getattr(self.settings_module, opt_name)
         if opt_name in self.defaults:
             return self.defaults[opt_name]
-        return super(CrawlerSettings, self).__getitem__(opt_name)
+        return Settings.__getitem__(self, opt_name)
 
     def __str__(self):
         return "<CrawlerSettings module=%r>" % self.settings_module
 
+CrawlerSettings = create_deprecated_class(
+    'CrawlerSettings', CrawlerSettings,
+    new_class_path='scrapy.settings.Settings')
+
 
 def iter_default_settings():
     """Return the default settings as an iterator of (name, value) tuples"""

scrapy/tests/test_cmdline/extensions.py

@@ -3,7 +3,7 @@
 class TestExtension(object):
 
     def __init__(self, settings):
-        settings.overrides['TEST1'] = "%s + %s" % (settings['TEST1'], 'started')
+        settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started'))
 
     @classmethod
     def from_crawler(cls, crawler):

scrapy/tests/test_downloadermiddleware_robotstxt.py

@@ -6,17 +6,17 @@ from twisted.trial import unittest
 from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
-from scrapy.settings import CrawlerSettings
+from scrapy.settings import Settings
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
 
     def test(self):
         crawler = mock.MagicMock()
-        crawler.settings = CrawlerSettings()
-        crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
+        crawler.settings = Settings()
+        crawler.settings.set('USER_AGENT', 'CustomAgent')
         self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
-        crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
+        crawler.settings.set('ROBOTSTXT_OBEY', True)
         crawler.engine.download = mock.MagicMock()
         ROBOTS = re.sub(r'^\s+(?m)', '', '''
         User-Agent: *

scrapy/tests/test_settings.py

@@ -1,82 +0,0 @@
-import unittest
-
-from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
-from scrapy.spider import Spider
-
-class SettingsTest(unittest.TestCase):
-
-    def test_get(self):
-        settings = Settings({
-            'TEST_ENABLED1': '1',
-            'TEST_ENABLED2': True,
-            'TEST_ENABLED3': 1,
-            'TEST_DISABLED1': '0',
-            'TEST_DISABLED2': False,
-            'TEST_DISABLED3': 0,
-            'TEST_INT1': 123,
-            'TEST_INT2': '123',
-            'TEST_FLOAT1': 123.45,
-            'TEST_FLOAT2': '123.45',
-            'TEST_LIST1': ['one', 'two'],
-            'TEST_LIST2': 'one,two',
-            'TEST_STR': 'value',
-            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
-            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
-        })
-        assert settings.getbool('TEST_ENABLED1') is True
-        assert settings.getbool('TEST_ENABLED2') is True
-        assert settings.getbool('TEST_ENABLED3') is True
-        assert settings.getbool('TEST_ENABLEDx') is False
-        assert settings.getbool('TEST_ENABLEDx', True) is True
-        assert settings.getbool('TEST_DISABLED1') is False
-        assert settings.getbool('TEST_DISABLED2') is False
-        assert settings.getbool('TEST_DISABLED3') is False
-        self.assertEqual(settings.getint('TEST_INT1'), 123)
-        self.assertEqual(settings.getint('TEST_INT2'), 123)
-        self.assertEqual(settings.getint('TEST_INTx'), 0)
-        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
-        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
-        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LISTx'), [])
-        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
-        self.assertEqual(settings['TEST_STR'], 'value')
-        self.assertEqual(settings.get('TEST_STR'), 'value')
-        self.assertEqual(settings['TEST_STRx'], None)
-        self.assertEqual(settings.get('TEST_STRx'), None)
-        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
-        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT3'), {})
-        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
-        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
-
-
-class CrawlerSettingsTest(unittest.TestCase):
-
-    def test_global_defaults(self):
-        crawler = get_crawler()
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 180)
-
-    def test_defaults(self):
-        crawler = get_crawler()
-        crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 99)
-
-    def test_settings_module(self):
-        crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 3)
-
-    def test_overrides(self):
-        crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
-        crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
-        self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 15)
-
-
-if __name__ == "__main__":
-    unittest.main()
-

scrapy/tests/test_settings/__init__.py

@@ -0,0 +1,195 @@
+import six
+import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from scrapy.settings import Settings, SettingsAttribute
+from . import default_settings
+
+
+class SettingsAttributeTest(unittest.TestCase):
+
+    def setUp(self):
+        self.attribute = SettingsAttribute('value', 10)
+
+    def test_set_greater_priority(self):
+        self.attribute.set('value2', 20)
+        self.assertEqual(self.attribute.value, 'value2')
+        self.assertEqual(self.attribute.priority, 20)
+
+    def test_set_equal_priority(self):
+        self.attribute.set('value2', 10)
+        self.assertEqual(self.attribute.value, 'value2')
+        self.assertEqual(self.attribute.priority, 10)
+
+    def test_set_less_priority(self):
+        self.attribute.set('value2', 0)
+        self.assertEqual(self.attribute.value, 'value')
+        self.assertEqual(self.attribute.priority, 10)
+
+
+class SettingsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.settings = Settings()
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
+    @mock.patch('scrapy.settings.default_settings', default_settings)
+    def test_initial_defaults(self):
+        settings = Settings()
+        self.assertEqual(len(settings.attributes), 1)
+        self.assertIn('TEST_DEFAULT', settings.attributes)
+
+        attr = settings.attributes['TEST_DEFAULT']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'defvalue')
+        self.assertEqual(attr.priority, 10)
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
+    @mock.patch('scrapy.settings.default_settings', {})
+    def test_initial_values(self):
+        settings = Settings({'TEST_OPTION': 'value'}, 10)
+        self.assertEqual(len(settings.attributes), 1)
+        self.assertIn('TEST_OPTION', settings.attributes)
+
+        attr = settings.attributes['TEST_OPTION']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.priority, 10)
+
+    def test_set_new_attribute(self):
+        self.settings.attributes = {}
+        self.settings.set('TEST_OPTION', 'value', 0)
+        self.assertIn('TEST_OPTION', self.settings.attributes)
+
+        attr = self.settings.attributes['TEST_OPTION']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.priority, 0)
+
+    def test_set_instance_identity_on_update(self):
+        attr = SettingsAttribute('value', 0)
+        self.settings.attributes = {'TEST_OPTION': attr}
+        self.settings.set('TEST_OPTION', 'othervalue', 10)
+
+        self.assertIn('TEST_OPTION', self.settings.attributes)
+        self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
+
+    def test_set_calls_settings_attributes_methods_on_update(self):
+        with mock.patch.object(SettingsAttribute, '__setattr__') as mock_setattr, \
+                mock.patch.object(SettingsAttribute, 'set') as mock_set:
+
+            attr = SettingsAttribute('value', 10)
+            self.settings.attributes = {'TEST_OPTION': attr}
+            mock_set.reset_mock()
+            mock_setattr.reset_mock()
+
+            for priority in (0, 10, 20):
+                self.settings.set('TEST_OPTION', 'othervalue', priority)
+                mock_set.assert_called_once_with('othervalue', priority)
+                self.assertFalse(mock_setattr.called)
+                mock_set.reset_mock()
+                mock_setattr.reset_mock()
+
+    def test_setdict_alias(self):
+        with mock.patch.object(self.settings, 'set') as mock_set:
+            self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
+            self.assertEqual(mock_set.call_count, 2)
+            calls = [mock.call('TEST_1', 'value1', 10),
+                     mock.call('TEST_2', 'value2', 10)]
+            mock_set.assert_has_calls(calls, any_order=True)
+
+    def test_setmodule_only_load_uppercase_vars(self):
+        class ModuleMock():
+            UPPERCASE_VAR = 'value'
+            MIXEDcase_VAR = 'othervalue'
+            lowercase_var = 'anothervalue'
+
+        self.settings.attributes = {}
+        self.settings.setmodule(ModuleMock(), 10)
+        self.assertIn('UPPERCASE_VAR', self.settings.attributes)
+        self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
+        self.assertNotIn('lowercase_var', self.settings.attributes)
+        self.assertEqual(len(self.settings.attributes), 1)
+
+    def test_setmodule_alias(self):
+        with mock.patch.object(self.settings, 'set') as mock_set:
+            self.settings.setmodule(default_settings, 10)
+            mock_set.assert_called_with('TEST_DEFAULT', 'defvalue', 10)
+
+    def test_setmodule_by_path(self):
+        self.settings.attributes = {}
+        self.settings.setmodule(default_settings, 10)
+        ctrl_attributes = self.settings.attributes.copy()
+
+        self.settings.attributes = {}
+        self.settings.setmodule(
+            'scrapy.tests.test_settings.default_settings', 10)
+
+        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+                              six.iterkeys(ctrl_attributes))
+
+        for attr, ctrl_attr in zip(six.itervalues(self.settings.attributes),
+                                   six.itervalues(ctrl_attributes)):
+            self.assertEqual(attr.value, ctrl_attr.value)
+            self.assertEqual(attr.priority, ctrl_attr.priority)
+
+    def test_get(self):
+        test_configuration = {
+            'TEST_ENABLED1': '1',
+            'TEST_ENABLED2': True,
+            'TEST_ENABLED3': 1,
+            'TEST_DISABLED1': '0',
+            'TEST_DISABLED2': False,
+            'TEST_DISABLED3': 0,
+            'TEST_INT1': 123,
+            'TEST_INT2': '123',
+            'TEST_FLOAT1': 123.45,
+            'TEST_FLOAT2': '123.45',
+            'TEST_LIST1': ['one', 'two'],
+            'TEST_LIST2': 'one,two',
+            'TEST_STR': 'value',
+            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
+            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
+        }
+        settings = self.settings
+        settings.attributes = {key: SettingsAttribute(value, 0) for key, value
+                               in six.iteritems(test_configuration)}
+
+        self.assertTrue(settings.getbool('TEST_ENABLED1'))
+        self.assertTrue(settings.getbool('TEST_ENABLED2'))
+        self.assertTrue(settings.getbool('TEST_ENABLED3'))
+        self.assertFalse(settings.getbool('TEST_ENABLEDx'))
+        self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
+        self.assertFalse(settings.getbool('TEST_DISABLED1'))
+        self.assertFalse(settings.getbool('TEST_DISABLED2'))
+        self.assertFalse(settings.getbool('TEST_DISABLED3'))
+        self.assertEqual(settings.getint('TEST_INT1'), 123)
+        self.assertEqual(settings.getint('TEST_INT2'), 123)
+        self.assertEqual(settings.getint('TEST_INTx'), 0)
+        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
+        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
+        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
+        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
+        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
+        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
+        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
+        self.assertEqual(settings.getlist('TEST_LISTx'), [])
+        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
+        self.assertEqual(settings['TEST_STR'], 'value')
+        self.assertEqual(settings.get('TEST_STR'), 'value')
+        self.assertEqual(settings['TEST_STRx'], None)
+        self.assertEqual(settings.get('TEST_STRx'), None)
+        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
+        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
+        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
+        self.assertEqual(settings.getdict('TEST_DICT3'), {})
+        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
+        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
+
+
+if __name__ == "__main__":
+    unittest.main()
+

scrapy/tests/test_settings/default_settings.py

@@ -0,0 +1,2 @@
+
+TEST_DEFAULT = 'defvalue'

scrapy/utils/project.py

@@ -6,7 +6,7 @@ from importlib import import_module
 from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
-from scrapy.settings import CrawlerSettings
+from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
@@ -53,20 +53,21 @@ def get_project_settings():
     if ENVVAR not in os.environ:
         project = os.environ.get('SCRAPY_PROJECT', 'default')
         init_env(project)
+
+    settings = Settings()
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
-        settings_module = import_module(settings_module_path)
-    else:
-        settings_module = None
-    settings = CrawlerSettings(settings_module)
+        settings.setmodule(settings_module_path, priority='project')
 
     # XXX: remove this hack
     pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-    settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
+    if pickled_settings:
+        settings.setdict(pickle.loads(pickled_settings), priority='project')
 
     # XXX: deprecate and remove this functionality
-    for k, v in os.environ.items():
-        if k.startswith('SCRAPY_'):
-            settings.overrides[k[7:]] = v
+    env_overrides = {k[7:]: v for k, v in os.environ.items() if
+                     k.startswith('SCRAPY_')}
+    if env_overrides:
+        settings.setdict(env_overrides, priority='project')
 
     return settings

scrapy/utils/test.py

@@ -22,20 +22,13 @@ def assert_aws_environ():
 
 def get_crawler(settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
-    will be used as the settings present in the settings module of the
-    CrawlerSettings.
+    will be used to populate the crawler settings with a project level
+    priority.
     """
     from scrapy.crawler import Crawler
-    from scrapy.settings import CrawlerSettings
+    from scrapy.settings import Settings
 
-    class SettingsModuleMock(object):
-        pass
-    settings_module = SettingsModuleMock()
-    if settings_dict:
-        for k, v in settings_dict.items():
-            setattr(settings_module, k, v)
-    settings = CrawlerSettings(settings_module)
-    return Crawler(settings)
+    return Crawler(Settings(settings_dict))
 
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this