diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 68bf68306..9f6297cb5 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,11 +6,10 @@ import inspect import scrapy from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree -from scrapy.conf import settings from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules -from scrapy.utils.project import inside_project +from scrapy.utils.project import inside_project, get_project_settings def _iter_command_classes(module_name): # TODO: add `name` attribute to commands and and merge this function with @@ -30,7 +29,7 @@ def _get_commands_from_module(module, inproject): d[cmdname] = cmd() return d -def _get_commands_dict(inproject): +def _get_commands_dict(settings, inproject): cmds = _get_commands_from_module('scrapy.commands', inproject) cmds_module = settings['COMMANDS_MODULE'] if cmds_module: @@ -45,19 +44,19 @@ def _pop_command_name(argv): return arg i += 1 -def _print_header(inproject): +def _print_header(settings, inproject): if inproject: print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ settings['BOT_NAME']) else: print "Scrapy %s - no active project\n" % scrapy.__version__ -def _print_commands(inproject): - _print_header(inproject) +def _print_commands(settings, inproject): + _print_header(settings, inproject) print "Usage:" print " scrapy [options] [args]\n" print "Available commands:" - cmds = _get_commands_dict(inproject) + cmds = _get_commands_dict(settings, inproject) for cmdname, cmdclass in sorted(cmds.iteritems()): print " %-13s %s" % (cmdname, cmdclass.short_desc()) if not inproject: @@ -66,8 +65,8 @@ def _print_commands(inproject): print print 'Use "scrapy -h" to see more info about a command' -def _print_unknown_command(cmdname, inproject): - _print_header(inproject) +def _print_unknown_command(settings, cmdname, inproject): + _print_header(settings, inproject) print "Unknown command: %s\n" % cmdname print 'Use "scrapy" to see available commands' @@ -84,18 +83,19 @@ def _run_print_help(parser, func, *a, **kw): def execute(argv=None): if argv is None: argv = sys.argv + settings = get_project_settings() crawler = CrawlerProcess(settings) crawler.install() inproject = inside_project() - cmds = _get_commands_dict(inproject) + cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ conflict_handler='resolve') if not cmdname: - _print_commands(inproject) + _print_commands(settings, inproject) sys.exit(0) elif cmdname not in cmds: - _print_unknown_command(cmdname, inproject) + _print_unknown_command(settings, cmdname, inproject) sys.exit(2) cmd = cmds[cmdname] diff --git a/scrapy/conf.py b/scrapy/conf.py index 795c8d7c0..b720ce60c 100644 --- a/scrapy/conf.py +++ b/scrapy/conf.py @@ -1,37 +1,7 @@ -""" -Scrapy settings manager - -See documentation in docs/topics/settings.rst -""" - -import os -import cPickle as pickle - -from scrapy.settings import CrawlerSettings -from scrapy.utils.conf import init_env - -ENVVAR = 'SCRAPY_SETTINGS_MODULE' - -def get_project_settings(): - if ENVVAR not in os.environ: - project = os.environ.get('SCRAPY_PROJECT', 'default') - init_env(project) - settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings') - try: - settings_module = __import__(settings_module_path, {}, {}, ['']) - except ImportError: - settings_module = None - settings = CrawlerSettings(settings_module) - - # XXX: remove this hack - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {} - - # XXX: deprecate and remove this functionality - for k, v in os.environ.items(): - if k.startswith('SCRAPY_'): - settings.overrides[k[7:]] = v - - return settings +# This module is kept for backwards compatibility. +# +# TODO: Add deprecation warning once all scrapy.conf instances have been +# removed from Scrapy codebase. +from scrapy.utils.project import get_project_settings settings = get_project_settings() diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 70ed4072d..fde7a4d09 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,15 +1,17 @@ +import os from os.path import join, dirname, abspath, isabs, exists -from os import makedirs, environ +import cPickle as pickle import warnings -from scrapy.utils.conf import closest_scrapy_cfg, get_config -from scrapy.utils.python import is_writable +from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env +from scrapy.settings import CrawlerSettings from scrapy.exceptions import NotConfigured +ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' def inside_project(): - scrapy_module = environ.get('SCRAPY_SETTINGS_MODULE') + scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') if scrapy_module is not None: try: __import__(scrapy_module) @@ -32,7 +34,7 @@ def project_data_dir(project='default'): raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") d = abspath(join(dirname(scrapy_cfg), '.scrapy')) if not exists(d): - makedirs(d) + os.makedirs(d) return d def data_path(path): @@ -40,3 +42,25 @@ def data_path(path): otherwise return the path unmodified """ return path if isabs(path) else join(project_data_dir(), path) + +def get_project_settings(): + if ENVVAR not in os.environ: + project = os.environ.get('SCRAPY_PROJECT', 'default') + init_env(project) + settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings') + try: + settings_module = __import__(settings_module_path, {}, {}, ['']) + except ImportError: + settings_module = None + settings = CrawlerSettings(settings_module) + + # XXX: remove this hack + pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") + settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {} + + # XXX: deprecate and remove this functionality + for k, v in os.environ.items(): + if k.startswith('SCRAPY_'): + settings.overrides[k[7:]] = v + + return settings