From 37776618a30cd50a80c71a27fbb9da05d0d8eca2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 2 Sep 2010 21:22:17 -0300 Subject: [PATCH] Changed format of scrapy.cfg file to contain a [settings] section and a 'default' key inside it, instead of the other way around --- examples/experimental/googledir/scrapy.cfg | 4 +-- examples/experimental/imdb/scrapy.cfg | 4 +-- examples/googledir/scrapy.cfg | 4 +-- scrapy/cmdline.py | 4 +-- scrapy/conf/__init__.py | 4 +-- scrapy/templates/project/scrapy.cfg | 4 +-- scrapy/utils/conf.py | 36 ++++++++++++++++------ 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/examples/experimental/googledir/scrapy.cfg b/examples/experimental/googledir/scrapy.cfg index d27f4ca95..164e66589 100644 --- a/examples/experimental/googledir/scrapy.cfg +++ b/examples/experimental/googledir/scrapy.cfg @@ -1,2 +1,2 @@ -[default] -settings = googledir.settings +[settings] +default = googledir.settings diff --git a/examples/experimental/imdb/scrapy.cfg b/examples/experimental/imdb/scrapy.cfg index a0a4adba9..fa607228e 100644 --- a/examples/experimental/imdb/scrapy.cfg +++ b/examples/experimental/imdb/scrapy.cfg @@ -1,2 +1,2 @@ -[default] -settings = imdb.settings +[settings] +default = imdb.settings diff --git a/examples/googledir/scrapy.cfg b/examples/googledir/scrapy.cfg index d27f4ca95..164e66589 100644 --- a/examples/googledir/scrapy.cfg +++ b/examples/googledir/scrapy.cfg @@ -1,2 +1,2 @@ -[default] -settings = googledir.settings +[settings] +default = googledir.settings diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 8d1a31d16..023541c15 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -87,8 +87,8 @@ def _check_deprecated_scrapy_ctl(argv, inproject): if not os.path.exists(cfg_path): with open(cfg_path, 'w') as f: f.write("# generated automatically - feel free to edit" + os.linesep) - f.write("[default]" + os.linesep) - f.write("settings = %s" % settings.settings_module_path + os.linesep) + f.write("[settings]" + os.linesep) + f.write("default = %s" % settings.settings_module_path + os.linesep) def _run_print_help(parser, func, *a, **kw): try: diff --git a/scrapy/conf/__init__.py b/scrapy/conf/__init__.py index ee442563f..e9f3fb136 100644 --- a/scrapy/conf/__init__.py +++ b/scrapy/conf/__init__.py @@ -8,7 +8,7 @@ import os import cPickle as pickle from scrapy.conf import default_settings -from scrapy.utils.conf import set_scrapy_settings_envvar +from scrapy.utils.conf import init_env import_ = lambda x: __import__(x, {}, {}, ['']) @@ -60,7 +60,7 @@ class EnvironmentSettings(Settings): self.disabled = os.environ.get('SCRAPY_SETTINGS_DISABLED', False) if self.ENVVAR not in os.environ: project = os.environ.get('SCRAPY_PROJECT', 'default') - set_scrapy_settings_envvar(project) + init_env(project) settings_module_path = os.environ.get(self.ENVVAR, 'scrapy_settings') self.set_settings_module(settings_module_path) diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg index d30c90567..7c357dc0d 100644 --- a/scrapy/templates/project/scrapy.cfg +++ b/scrapy/templates/project/scrapy.cfg @@ -1,2 +1,2 @@ -[default] -settings = ${project_name}.settings +[settings] +default = ${project_name}.settings diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0e21fa798..0ec409903 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,6 +1,6 @@ import sys import os -from ConfigParser import RawConfigParser +from ConfigParser import SafeConfigParser from operator import itemgetter def build_component_list(base, custom): @@ -17,10 +17,14 @@ def build_component_list(base, custom): def arglist_to_dict(arglist): """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a - dict""" + dict + """ return dict(x.split('=', 1) for x in arglist) def closest_scrapy_cfg(path='.', prevpath=None): + """Return the path to the closest scrapy.cfg file by traversing the current + directory and its parents + """ if path == prevpath: return '' path = os.path.abspath(path) @@ -29,13 +33,25 @@ def closest_scrapy_cfg(path='.', prevpath=None): return cfgfile return closest_scrapy_cfg(os.path.dirname(path), path) -def set_scrapy_settings_envvar(project='default', set_syspath=True): - scrapy_cfg = closest_scrapy_cfg() - cfg_sources = [scrapy_cfg, os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg'] - cfg = RawConfigParser() - cfg.read(cfg_sources) - if cfg.has_option(project, 'settings'): - os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get(project, 'settings') - projdir = os.path.dirname(scrapy_cfg) +def init_env(project='default', set_syspath=True): + """Initialize environment to use command-line tool from inside a project + dir. This sets the Scrapy settings module and modifies the Python path to + be able to locate the project module. + """ + cfg = get_config() + if cfg.has_option('settings', project): + os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project) + closest = closest_scrapy_cfg() + if closest: + projdir = os.path.dirname(closest) if set_syspath and projdir not in sys.path: sys.path.append(projdir) + +def get_config(use_closest=True): + """Get Scrapy config file as a SafeConfigParser""" + sources = [os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg'] + if use_closest: + sources.insert(0, closest_scrapy_cfg()) + cfg = SafeConfigParser() + cfg.read(sources) + return cfg