removed another instance of scrapy.conf.settings singleton, this time from scrapy.cmdline (which manages scrapy command line tool), by moving the get_project_settings() function to scrapy.utils.project

scrapy/cmdline.py

@@ -6,11 +6,10 @@ import inspect
 import scrapy
 from scrapy.crawler import CrawlerProcess
 from scrapy.xlib import lsprofcalltree
-from scrapy.conf import settings
 from scrapy.command import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
-from scrapy.utils.project import inside_project
+from scrapy.utils.project import inside_project, get_project_settings
 
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
@@ -30,7 +29,7 @@ def _get_commands_from_module(module, inproject):
             d[cmdname] = cmd()
     return d
 
-def _get_commands_dict(inproject):
+def _get_commands_dict(settings, inproject):
     cmds = _get_commands_from_module('scrapy.commands', inproject)
     cmds_module = settings['COMMANDS_MODULE']
     if cmds_module:
@@ -45,19 +44,19 @@ def _pop_command_name(argv):
             return arg
         i += 1
 
-def _print_header(inproject):
+def _print_header(settings, inproject):
     if inproject:
         print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
             settings['BOT_NAME'])
     else:
         print "Scrapy %s - no active project\n" % scrapy.__version__
 
-def _print_commands(inproject):
-    _print_header(inproject)
+def _print_commands(settings, inproject):
+    _print_header(settings, inproject)
     print "Usage:"
     print "  scrapy <command> [options] [args]\n"
     print "Available commands:"
-    cmds = _get_commands_dict(inproject)
+    cmds = _get_commands_dict(settings, inproject)
     for cmdname, cmdclass in sorted(cmds.iteritems()):
         print "  %-13s %s" % (cmdname, cmdclass.short_desc())
     if not inproject:
@@ -66,8 +65,8 @@ def _print_commands(inproject):
     print
     print 'Use "scrapy <command> -h" to see more info about a command'
 
-def _print_unknown_command(cmdname, inproject):
-    _print_header(inproject)
+def _print_unknown_command(settings, cmdname, inproject):
+    _print_header(settings, inproject)
     print "Unknown command: %s\n" % cmdname
     print 'Use "scrapy" to see available commands' 
 
@@ -84,18 +83,19 @@ def _run_print_help(parser, func, *a, **kw):
 def execute(argv=None):
     if argv is None:
         argv = sys.argv
+    settings = get_project_settings()
     crawler = CrawlerProcess(settings)
     crawler.install()
     inproject = inside_project()
-    cmds = _get_commands_dict(inproject)
+    cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
     parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
         conflict_handler='resolve')
     if not cmdname:
-        _print_commands(inproject)
+        _print_commands(settings, inproject)
         sys.exit(0)
     elif cmdname not in cmds:
-        _print_unknown_command(cmdname, inproject)
+        _print_unknown_command(settings, cmdname, inproject)
         sys.exit(2)
 
     cmd = cmds[cmdname]

scrapy/conf.py

@@ -1,37 +1,7 @@
-"""
-Scrapy settings manager
-
-See documentation in docs/topics/settings.rst
-"""
-
-import os
-import cPickle as pickle
-
-from scrapy.settings import CrawlerSettings
-from scrapy.utils.conf import init_env
-
-ENVVAR = 'SCRAPY_SETTINGS_MODULE'
-
-def get_project_settings():
-    if ENVVAR not in os.environ:
-        project = os.environ.get('SCRAPY_PROJECT', 'default')
-        init_env(project)
-    settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings')
-    try:
-        settings_module = __import__(settings_module_path, {}, {}, [''])
-    except ImportError:
-        settings_module = None
-    settings = CrawlerSettings(settings_module)
-
-    # XXX: remove this hack
-    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-    settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
-
-    # XXX: deprecate and remove this functionality
-    for k, v in os.environ.items():
-        if k.startswith('SCRAPY_'):
-            settings.overrides[k[7:]] = v
-
-    return settings
+# This module is kept for backwards compatibility.
+#
+# TODO: Add deprecation warning once all scrapy.conf instances have been
+# removed from Scrapy codebase.
 
+from scrapy.utils.project import get_project_settings
 settings = get_project_settings()

scrapy/utils/project.py

@@ -1,15 +1,17 @@
+import os
 from os.path import join, dirname, abspath, isabs, exists
-from os import makedirs, environ
+import cPickle as pickle
 import warnings
 
-from scrapy.utils.conf import closest_scrapy_cfg, get_config
-from scrapy.utils.python import is_writable
+from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
+from scrapy.settings import CrawlerSettings
 from scrapy.exceptions import NotConfigured
 
+ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
 
 def inside_project():
-    scrapy_module = environ.get('SCRAPY_SETTINGS_MODULE')
+    scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
         try:
             __import__(scrapy_module)
@@ -32,7 +34,7 @@ def project_data_dir(project='default'):
             raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
         d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
     if not exists(d):
-        makedirs(d)
+        os.makedirs(d)
     return d
 
 def data_path(path):
@@ -40,3 +42,25 @@ def data_path(path):
     otherwise return the path unmodified
     """
     return path if isabs(path) else join(project_data_dir(), path)
+
+def get_project_settings():
+    if ENVVAR not in os.environ:
+        project = os.environ.get('SCRAPY_PROJECT', 'default')
+        init_env(project)
+    settings_module_path = os.environ.get(ENVVAR, 'scrapy_settings')
+    try:
+        settings_module = __import__(settings_module_path, {}, {}, [''])
+    except ImportError:
+        settings_module = None
+    settings = CrawlerSettings(settings_module)
+
+    # XXX: remove this hack
+    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
+    settings.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
+
+    # XXX: deprecate and remove this functionality
+    for k, v in os.environ.items():
+        if k.startswith('SCRAPY_'):
+            settings.overrides[k[7:]] = v
+
+    return settings