Splitted settings classes from settings singleton. Closes #244

--HG--
rename : scrapy/conf/__init__.py => scrapy/conf.py
rename : scrapy/conf/default_settings.py => scrapy/settings/default_settings.py
rename : scrapy/tests/test_conf.py => scrapy/tests/test_settings.py

docs/topics/settings.rst

@@ -39,10 +39,9 @@ different precedence. Here is the list of them in decreasing order of
 precedence:
 
  1. Global overrides (most precedence)
- 2. Environment variables
- 3. scrapy_settings
- 4. Default settings per-command
- 5. Default global settings (less precedence)
+ 2. Project settings module
+ 3. Default settings per-command
+ 4. Default global settings (less precedence)
 
 These mechanisms are described in more detail below.
 
@@ -65,27 +64,14 @@ Example::
 
     scrapy crawl domain.com --set LOG_FILE=scrapy.log
 
-2. Environment variables
-------------------------
+2. Project settings module
+--------------------------
 
-You can populate settings using environment variables prefixed with
-``SCRAPY_``. For example, to change the log file location un Unix systems::
+The project settings module is the standard configuration file for your Scrapy
+project.  It's where most of your custom settings will be populated. For
+example:: ``myproject.settings``.
 
-    $ export SCRAPY_LOG_FILE=scrapy.log
-    $ scrapy crawl example.com
-
-In Windows systems, you can change the environment variables from the Control
-Panel following `these guidelines`_.
-
-.. _these guidelines: http://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
-
-3. scrapy_settings
-------------------
-
-scrapy_settings is the standard configuration file for your Scrapy project.
-It's where most of your custom settings will be populated.
-
-4. Default settings per-command
+3. Default settings per-command
 -------------------------------
 
 Each :doc:`Scrapy tool </topics/commands>` command can have its own default
@@ -93,11 +79,11 @@ settings, which override the global default settings. Those custom command
 settings are specified in the ``default_settings`` attribute of the command
 class.
 
-5. Default global settings
+4. Default global settings
 --------------------------
 
-The global defaults are located in scrapy.conf.default_settings and documented
-in the :ref:`topics-settings-ref` section.
+The global defaults are located in the ``scrapy.settings.default_settings``
+module and documented in the :ref:`topics-settings-ref` section.
 
 How to access settings
 ======================

examples/experimental/googledir/googledir/settings.py

@@ -4,11 +4,6 @@
 # default. All the other settings are documented here:
 #
 #     http://doc.scrapy.org/topics/settings.html
-#
-# Or you can copy and paste them from where they're defined in Scrapy:
-# 
-#     scrapy/conf/default_settings.py
-#
 
 BOT_NAME = 'googledir'
 BOT_VERSION = '1.0'

examples/experimental/imdb/imdb/settings.py

@@ -4,11 +4,6 @@
 # default. All the other settings are documented here:
 #
 #     http://doc.scrapy.org/topics/settings.html
-#
-# Or you can copy and paste them from where they're defined in Scrapy:
-# 
-#     scrapy/conf/default_settings.py
-#
 
 BOT_NAME = 'imdb'
 BOT_VERSION = '1.0'

scrapy/conf.py

@@ -0,0 +1,40 @@
+"""
+Scrapy settings manager
+
+See documentation in docs/topics/settings.rst
+"""
+
+import os
+import cPickle as pickle
+
+from scrapy.settings import CrawlerSettings
+from scrapy.utils.conf import init_env
+
+ENVVAR = 'SCRAPY_SETTINGS_MODULE'
+
+def get_project_settings():
+    if ENVVAR not in os.environ:
+        project = os.environ.get('SCRAPY_PROJECT', 'default')
+        init_env(project)
+    settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings')
+    try:
+        settings_module = __import__(settings_module_path, {}, {}, [''])
+    except ImportError:
+        settings_module = None
+    settings = CrawlerSettings(settings_module)
+
+    # XXX: remove this hack
+    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
+    settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
+
+    # XXX: deprecate and remove this functionality
+    for k, v in os.environ.items():
+        if k.startswith('SCRAPY_'):
+            settings.overrides[k[7:]] = v
+
+    return settings
+
+if os.environ.get('SCRAPY_SETTINGS_DISABLED'):
+    settings = CrawlerSettings()
+else:
+    settings = get_project_settings()

scrapy/conf/__init__.py

@@ -1,94 +0,0 @@
-"""
-Scrapy settings manager
-
-See documentation in docs/topics/settings.rst
-"""
-
-import os
-import cPickle as pickle
-
-from scrapy.conf import default_settings
-from scrapy.utils.conf import init_env
-
-import_ = lambda x: __import__(x, {}, {}, [''])
-
-
-class Settings(object):
-
-    def __init__(self, values=None):
-        self.values = values.copy() if values else {}
-        self.global_defaults = default_settings
-
-    def __getitem__(self, opt_name):
-        if opt_name in self.values:
-            return self.values[opt_name]
-        return getattr(self.global_defaults, opt_name, None)
-
-    def get(self, name, default=None):
-        return self[name] if self[name] is not None else default
-
-    def getbool(self, name, default=False):
-        """
-        True is: 1, '1', True
-        False is: 0, '0', False, None
-        """
-        return bool(int(self.get(name, default)))
-
-    def getint(self, name, default=0):
-        return int(self.get(name, default))
-
-    def getfloat(self, name, default=0.0):
-        return float(self.get(name, default))
-
-    def getlist(self, name, default=None):
-        value = self.get(name)
-        if value is None:
-            return default or []
-        elif hasattr(value, '__iter__'):
-            return value
-        else:
-            return str(value).split(',')
-
-
-class EnvironmentSettings(Settings):
-
-    ENVVAR = 'SCRAPY_SETTINGS_MODULE'
-
-    def __init__(self):
-        super(EnvironmentSettings, self).__init__()
-        self.defaults = {}
-        self.disabled = os.environ.get('SCRAPY_SETTINGS_DISABLED', False)
-        if self.ENVVAR not in os.environ:
-            project = os.environ.get('SCRAPY_PROJECT', 'default')
-            init_env(project)
-        settings_module_path = os.environ.get(self.ENVVAR, 'scrapy_settings')
-        self.set_settings_module(settings_module_path)
-
-        # XXX: find a better solution for this hack
-        pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-        self.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
-
-    def set_settings_module(self, settings_module_path):
-        self.settings_module_path = settings_module_path
-        try:
-            self.settings_module = import_(settings_module_path)
-        except ImportError:
-            self.settings_module = None
-
-    def __getitem__(self, opt_name):
-        if not self.disabled:
-            if opt_name in self.overrides:
-                return self.overrides[opt_name]
-            if 'SCRAPY_' + opt_name in os.environ:
-                return os.environ['SCRAPY_' + opt_name]
-            if hasattr(self.settings_module, opt_name):
-                return getattr(self.settings_module, opt_name)
-            if opt_name in self.defaults:
-                return self.defaults[opt_name]
-        return super(EnvironmentSettings, self).__getitem__(opt_name)
-
-    def __str__(self):
-        return "<Settings %r>" % self.settings_module_path
-
-
-settings = EnvironmentSettings()

scrapy/contrib/downloadermiddleware/retry.py

@@ -11,9 +11,6 @@ once the spider has finished crawling all regular (non failed) pages. Once
 there is no more failed pages to retry this middleware sends a signal
 (retry_complete), so other extensions could connect to that signal.
 
-Default values are located in scrapy.conf.default_settings, like any other
-setting
-
 About HTTP errors to consider:
 
 - You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP

scrapy/settings/__init__.py

@@ -0,0 +1,59 @@
+from . import default_settings
+
+
+class Settings(object):
+
+    def __init__(self, values=None):
+        self.values = values.copy() if values else {}
+        self.global_defaults = default_settings
+
+    def __getitem__(self, opt_name):
+        if opt_name in self.values:
+            return self.values[opt_name]
+        return getattr(self.global_defaults, opt_name, None)
+
+    def get(self, name, default=None):
+        return self[name] if self[name] is not None else default
+
+    def getbool(self, name, default=False):
+        """
+        True is: 1, '1', True
+        False is: 0, '0', False, None
+        """
+        return bool(int(self.get(name, default)))
+
+    def getint(self, name, default=0):
+        return int(self.get(name, default))
+
+    def getfloat(self, name, default=0.0):
+        return float(self.get(name, default))
+
+    def getlist(self, name, default=None):
+        value = self.get(name)
+        if value is None:
+            return default or []
+        elif hasattr(value, '__iter__'):
+            return value
+        else:
+            return str(value).split(',')
+
+
+class CrawlerSettings(Settings):
+
+    def __init__(self, settings_module=None, **kw):
+        super(CrawlerSettings, self).__init__(**kw)
+        self.settings_module = settings_module
+        self.overrides = {}
+        self.defaults = {}
+
+    def __getitem__(self, opt_name):
+        if opt_name in self.overrides:
+            return self.overrides[opt_name]
+        if self.settings_module and hasattr(self.settings_module, opt_name):
+            return getattr(self.settings_module, opt_name)
+        if opt_name in self.defaults:
+            return self.defaults[opt_name]
+        return super(CrawlerSettings, self).__getitem__(opt_name)
+
+    def __str__(self):
+        return "<CrawlerSettings module=%r>" % self.settings_module.__name__

scrapy/shell.py

@@ -18,7 +18,8 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
 from scrapy.utils.url import any_to_uri
 from scrapy.utils.console import start_python_console
-from scrapy.conf import settings, Settings
+from scrapy.conf import settings
+from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse
 
 class Shell(object):

scrapy/templates/project/module/settings.py.tmpl

@@ -5,10 +5,6 @@
 #
 #     http://doc.scrapy.org/topics/settings.html
 #
-# Or you can copy and paste them from where they're defined in Scrapy:
-# 
-#     scrapy/conf/default_settings.py
-#
 
 BOT_NAME = '$project_name'
 BOT_VERSION = '1.0'

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -3,7 +3,7 @@ import unittest, tempfile, shutil, time
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spider import BaseSpider
 from scrapy.contrib.downloadermiddleware.httpcache import FilesystemCacheStorage, HttpCacheMiddleware
-from scrapy.conf import Settings
+from scrapy.settings import Settings
 from scrapy.exceptions import IgnoreRequest
 
 

scrapy/tests/test_engine.py

@@ -17,7 +17,7 @@ from twisted.web import server, static, util
 from twisted.trial import unittest
 
 from scrapy import signals
-from scrapy.conf import Settings
+from scrapy.settings import Settings
 from scrapy.crawler import Crawler
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.tests import tests_datadir

scrapy/tests/test_middleware.py

@@ -1,6 +1,6 @@
 from twisted.trial import unittest
 
-from scrapy.conf import Settings
+from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
 

scrapy/tests/test_pipeline_media.py

@@ -2,7 +2,7 @@ from twisted.trial import unittest
 from twisted.python import failure
 from twisted.internet import defer, reactor
 
-from scrapy.conf import Settings
+from scrapy.settings import Settings
 from scrapy.crawler import Crawler
 from scrapy.http import Request, Response
 from scrapy.spider import BaseSpider

scrapy/tests/test_settings.py

@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.conf import Settings
+from scrapy.settings import Settings
 
 class SettingsTest(unittest.TestCase):