better backwards compatibility for scrapy.conf.settings

Althought this backwards compatibility is more complex, it avoid modules
from failing when importing scrapy.conf, if they are not run through
"scrapy" command (such as when running tests on scrapy projects code).

scrapy/cmdline.py

@@ -96,6 +96,15 @@ def execute(argv=None, settings=None):
         settings = get_project_settings()
     check_deprecated_settings(settings)
 
+    # --- backwards compatibility for scrapy.conf.settings singleton ---
+    import warnings
+    from scrapy.exceptions import ScrapyDeprecationWarning
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+        from scrapy import conf
+        conf.settings = settings
+    # ------------------------------------------------------------------
+
     crawler = CrawlerProcess(settings)
     crawler.install()
     inproject = inside_project()

scrapy/conf.py

@@ -1,8 +1,11 @@
 # This module is kept for backwards compatibility, so users can import
 # scrapy.conf.settings and get the settings they expect
 
-from scrapy.project import crawler
-settings = crawler.settings
+import sys
+
+if 'scrapy.cmdline' not in sys.modules:
+    from scrapy.utils.project import get_project_settings
+    settings = get_project_settings()
 
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning