From b6c2b55e5b2d5ff2bb6d13d46d27cd5f273f6cdc Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 22 Sep 2010 15:47:33 -0300 Subject: [PATCH] Splitted settings classes from settings singleton. Closes #244 --HG-- rename : scrapy/conf/__init__.py => scrapy/conf.py rename : scrapy/conf/default_settings.py => scrapy/settings/default_settings.py rename : scrapy/tests/test_conf.py => scrapy/tests/test_settings.py --- docs/topics/settings.rst | 38 +++----- .../googledir/googledir/settings.py | 5 - examples/experimental/imdb/imdb/settings.py | 5 - scrapy/conf.py | 40 ++++++++ scrapy/conf/__init__.py | 94 ------------------- scrapy/contrib/downloadermiddleware/retry.py | 3 - scrapy/settings/__init__.py | 59 ++++++++++++ scrapy/{conf => settings}/default_settings.py | 0 scrapy/shell.py | 3 +- .../templates/project/module/settings.py.tmpl | 4 - .../test_downloadermiddleware_httpcache.py | 2 +- scrapy/tests/test_engine.py | 2 +- scrapy/tests/test_middleware.py | 2 +- scrapy/tests/test_pipeline_media.py | 2 +- .../tests/{test_conf.py => test_settings.py} | 2 +- 15 files changed, 118 insertions(+), 143 deletions(-) create mode 100644 scrapy/conf.py delete mode 100644 scrapy/conf/__init__.py create mode 100644 scrapy/settings/__init__.py rename scrapy/{conf => settings}/default_settings.py (100%) rename scrapy/tests/{test_conf.py => test_settings.py} (98%) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2f32be53e..a63eb1f62 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -39,10 +39,9 @@ different precedence. Here is the list of them in decreasing order of precedence: 1. Global overrides (most precedence) - 2. Environment variables - 3. scrapy_settings - 4. Default settings per-command - 5. Default global settings (less precedence) + 2. Project settings module + 3. Default settings per-command + 4. Default global settings (less precedence) These mechanisms are described in more detail below. @@ -65,27 +64,14 @@ Example:: scrapy crawl domain.com --set LOG_FILE=scrapy.log -2. Environment variables ------------------------- +2. Project settings module +-------------------------- -You can populate settings using environment variables prefixed with -``SCRAPY_``. For example, to change the log file location un Unix systems:: +The project settings module is the standard configuration file for your Scrapy +project. It's where most of your custom settings will be populated. For +example:: ``myproject.settings``. - $ export SCRAPY_LOG_FILE=scrapy.log - $ scrapy crawl example.com - -In Windows systems, you can change the environment variables from the Control -Panel following `these guidelines`_. - -.. _these guidelines: http://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx - -3. scrapy_settings ------------------- - -scrapy_settings is the standard configuration file for your Scrapy project. -It's where most of your custom settings will be populated. - -4. Default settings per-command +3. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -93,11 +79,11 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +4. Default global settings -------------------------- -The global defaults are located in scrapy.conf.default_settings and documented -in the :ref:`topics-settings-ref` section. +The global defaults are located in the ``scrapy.settings.default_settings`` +module and documented in the :ref:`topics-settings-ref` section. How to access settings ====================== diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py index 4e3c11163..38f1df2a5 100644 --- a/examples/experimental/googledir/googledir/settings.py +++ b/examples/experimental/googledir/googledir/settings.py @@ -4,11 +4,6 @@ # default. All the other settings are documented here: # # http://doc.scrapy.org/topics/settings.html -# -# Or you can copy and paste them from where they're defined in Scrapy: -# -# scrapy/conf/default_settings.py -# BOT_NAME = 'googledir' BOT_VERSION = '1.0' diff --git a/examples/experimental/imdb/imdb/settings.py b/examples/experimental/imdb/imdb/settings.py index de026dc14..e0a8db52a 100644 --- a/examples/experimental/imdb/imdb/settings.py +++ b/examples/experimental/imdb/imdb/settings.py @@ -4,11 +4,6 @@ # default. All the other settings are documented here: # # http://doc.scrapy.org/topics/settings.html -# -# Or you can copy and paste them from where they're defined in Scrapy: -# -# scrapy/conf/default_settings.py -# BOT_NAME = 'imdb' BOT_VERSION = '1.0' diff --git a/scrapy/conf.py b/scrapy/conf.py new file mode 100644 index 000000000..c47c38626 --- /dev/null +++ b/scrapy/conf.py @@ -0,0 +1,40 @@ +""" +Scrapy settings manager + +See documentation in docs/topics/settings.rst +""" + +import os +import cPickle as pickle + +from scrapy.settings import CrawlerSettings +from scrapy.utils.conf import init_env + +ENVVAR = 'SCRAPY_SETTINGS_MODULE' + +def get_project_settings(): + if ENVVAR not in os.environ: + project = os.environ.get('SCRAPY_PROJECT', 'default') + init_env(project) + settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings') + try: + settings_module = __import__(settings_module_path, {}, {}, ['']) + except ImportError: + settings_module = None + settings = CrawlerSettings(settings_module) + + # XXX: remove this hack + pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") + settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {} + + # XXX: deprecate and remove this functionality + for k, v in os.environ.items(): + if k.startswith('SCRAPY_'): + settings.overrides[k[7:]] = v + + return settings + +if os.environ.get('SCRAPY_SETTINGS_DISABLED'): + settings = CrawlerSettings() +else: + settings = get_project_settings() diff --git a/scrapy/conf/__init__.py b/scrapy/conf/__init__.py deleted file mode 100644 index e9f3fb136..000000000 --- a/scrapy/conf/__init__.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Scrapy settings manager - -See documentation in docs/topics/settings.rst -""" - -import os -import cPickle as pickle - -from scrapy.conf import default_settings -from scrapy.utils.conf import init_env - -import_ = lambda x: __import__(x, {}, {}, ['']) - - -class Settings(object): - - def __init__(self, values=None): - self.values = values.copy() if values else {} - self.global_defaults = default_settings - - def __getitem__(self, opt_name): - if opt_name in self.values: - return self.values[opt_name] - return getattr(self.global_defaults, opt_name, None) - - def get(self, name, default=None): - return self[name] if self[name] is not None else default - - def getbool(self, name, default=False): - """ - True is: 1, '1', True - False is: 0, '0', False, None - """ - return bool(int(self.get(name, default))) - - def getint(self, name, default=0): - return int(self.get(name, default)) - - def getfloat(self, name, default=0.0): - return float(self.get(name, default)) - - def getlist(self, name, default=None): - value = self.get(name) - if value is None: - return default or [] - elif hasattr(value, '__iter__'): - return value - else: - return str(value).split(',') - - -class EnvironmentSettings(Settings): - - ENVVAR = 'SCRAPY_SETTINGS_MODULE' - - def __init__(self): - super(EnvironmentSettings, self).__init__() - self.defaults = {} - self.disabled = os.environ.get('SCRAPY_SETTINGS_DISABLED', False) - if self.ENVVAR not in os.environ: - project = os.environ.get('SCRAPY_PROJECT', 'default') - init_env(project) - settings_module_path = os.environ.get(self.ENVVAR, 'scrapy_settings') - self.set_settings_module(settings_module_path) - - # XXX: find a better solution for this hack - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - self.overrides = pickle.loads(pickled_settings) if pickled_settings else {} - - def set_settings_module(self, settings_module_path): - self.settings_module_path = settings_module_path - try: - self.settings_module = import_(settings_module_path) - except ImportError: - self.settings_module = None - - def __getitem__(self, opt_name): - if not self.disabled: - if opt_name in self.overrides: - return self.overrides[opt_name] - if 'SCRAPY_' + opt_name in os.environ: - return os.environ['SCRAPY_' + opt_name] - if hasattr(self.settings_module, opt_name): - return getattr(self.settings_module, opt_name) - if opt_name in self.defaults: - return self.defaults[opt_name] - return super(EnvironmentSettings, self).__getitem__(opt_name) - - def __str__(self): - return "" % self.settings_module_path - - -settings = EnvironmentSettings() diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index 854dbb56a..9c0a2c788 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -11,9 +11,6 @@ once the spider has finished crawling all regular (non failed) pages. Once there is no more failed pages to retry this middleware sends a signal (retry_complete), so other extensions could connect to that signal. -Default values are located in scrapy.conf.default_settings, like any other -setting - About HTTP errors to consider: - You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py new file mode 100644 index 000000000..30b80a1cf --- /dev/null +++ b/scrapy/settings/__init__.py @@ -0,0 +1,59 @@ +from . import default_settings + + +class Settings(object): + + def __init__(self, values=None): + self.values = values.copy() if values else {} + self.global_defaults = default_settings + + def __getitem__(self, opt_name): + if opt_name in self.values: + return self.values[opt_name] + return getattr(self.global_defaults, opt_name, None) + + def get(self, name, default=None): + return self[name] if self[name] is not None else default + + def getbool(self, name, default=False): + """ + True is: 1, '1', True + False is: 0, '0', False, None + """ + return bool(int(self.get(name, default))) + + def getint(self, name, default=0): + return int(self.get(name, default)) + + def getfloat(self, name, default=0.0): + return float(self.get(name, default)) + + def getlist(self, name, default=None): + value = self.get(name) + if value is None: + return default or [] + elif hasattr(value, '__iter__'): + return value + else: + return str(value).split(',') + + +class CrawlerSettings(Settings): + + def __init__(self, settings_module=None, **kw): + super(CrawlerSettings, self).__init__(**kw) + self.settings_module = settings_module + self.overrides = {} + self.defaults = {} + + def __getitem__(self, opt_name): + if opt_name in self.overrides: + return self.overrides[opt_name] + if self.settings_module and hasattr(self.settings_module, opt_name): + return getattr(self.settings_module, opt_name) + if opt_name in self.defaults: + return self.defaults[opt_name] + return super(CrawlerSettings, self).__getitem__(opt_name) + + def __str__(self): + return "" % self.settings_module.__name__ diff --git a/scrapy/conf/default_settings.py b/scrapy/settings/default_settings.py similarity index 100% rename from scrapy/conf/default_settings.py rename to scrapy/settings/default_settings.py diff --git a/scrapy/shell.py b/scrapy/shell.py index 12947f480..a4f4594b0 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -18,7 +18,8 @@ from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.url import any_to_uri from scrapy.utils.console import start_python_console -from scrapy.conf import settings, Settings +from scrapy.conf import settings +from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse class Shell(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 07f576f91..ce2751f0f 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,10 +5,6 @@ # # http://doc.scrapy.org/topics/settings.html # -# Or you can copy and paste them from where they're defined in Scrapy: -# -# scrapy/conf/default_settings.py -# BOT_NAME = '$project_name' BOT_VERSION = '1.0' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 4e569ea7a..ae6339208 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -3,7 +3,7 @@ import unittest, tempfile, shutil, time from scrapy.http import Response, HtmlResponse, Request from scrapy.spider import BaseSpider from scrapy.contrib.downloadermiddleware.httpcache import FilesystemCacheStorage, HttpCacheMiddleware -from scrapy.conf import Settings +from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 6549fefa6..8f4ab8fb7 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -17,7 +17,7 @@ from twisted.web import server, static, util from twisted.trial import unittest from scrapy import signals -from scrapy.conf import Settings +from scrapy.settings import Settings from scrapy.crawler import Crawler from scrapy.xlib.pydispatch import dispatcher from scrapy.tests import tests_datadir diff --git a/scrapy/tests/test_middleware.py b/scrapy/tests/test_middleware.py index 1eca89b27..145b86b02 100644 --- a/scrapy/tests/test_middleware.py +++ b/scrapy/tests/test_middleware.py @@ -1,6 +1,6 @@ from twisted.trial import unittest -from scrapy.conf import Settings +from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager diff --git a/scrapy/tests/test_pipeline_media.py b/scrapy/tests/test_pipeline_media.py index 6877fcfad..a9731e6bd 100644 --- a/scrapy/tests/test_pipeline_media.py +++ b/scrapy/tests/test_pipeline_media.py @@ -2,7 +2,7 @@ from twisted.trial import unittest from twisted.python import failure from twisted.internet import defer, reactor -from scrapy.conf import Settings +from scrapy.settings import Settings from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.spider import BaseSpider diff --git a/scrapy/tests/test_conf.py b/scrapy/tests/test_settings.py similarity index 98% rename from scrapy/tests/test_conf.py rename to scrapy/tests/test_settings.py index 9eddb0206..dfccf5dff 100644 --- a/scrapy/tests/test_conf.py +++ b/scrapy/tests/test_settings.py @@ -1,6 +1,6 @@ import unittest -from scrapy.conf import Settings +from scrapy.settings import Settings class SettingsTest(unittest.TestCase):