Merge pull request #4375 from Gallaecio/env-warn-whitelist

Stop deprecation warnings on arbitrary SCRAPY-prefixed env vars

scrapy/utils/project.py

@@ -75,9 +75,24 @@ def get_project_settings():
                       "is deprecated.", ScrapyDeprecationWarning)
         settings.setdict(pickle.loads(pickled_settings), priority='project')
 
-    env_overrides = {k[7:]: v for k, v in os.environ.items() if
-                     k.startswith('SCRAPY_')}
-    if env_overrides:
-        warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning)
-        settings.setdict(env_overrides, priority='project')
+    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
+                      k.startswith('SCRAPY_')}
+    valid_envvars = {
+        'CHECK',
+        'PICKLED_SETTINGS_TO_OVERRIDE',
+        'PROJECT',
+        'PYTHON_SHELL',
+        'SETTINGS_MODULE',
+    }
+    setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars}
+    if setting_envvars:
+        setting_envvar_list = ', '.join(sorted(setting_envvars))
+        warnings.warn(
+            'Use of environment variables prefixed with SCRAPY_ to override '
+            'settings is deprecated. The following environment variables are '
+            'currently defined: {}'.format(setting_envvar_list),
+            ScrapyDeprecationWarning
+        )
+    settings.setdict(scrapy_envvars, priority='project')
+
     return settings

tests/test_utils_project.py

@@ -3,7 +3,11 @@ import os
 import tempfile
 import shutil
 import contextlib
-from scrapy.utils.project import data_path
+
+from pytest import warns
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.project import data_path, get_project_settings
 
 
 @contextlib.contextmanager
@@ -41,3 +45,53 @@ class ProjectUtilsTest(unittest.TestCase):
             )
             abspath = os.path.join(os.path.sep, 'absolute', 'path')
             self.assertEqual(abspath, data_path(abspath))
+
+
+@contextlib.contextmanager
+def set_env(**update):
+    modified = set(update.keys()) & set(os.environ.keys())
+    update_after = {k: os.environ[k] for k in modified}
+    remove_after = frozenset(k for k in update if k not in os.environ)
+    try:
+        os.environ.update(update)
+        yield
+    finally:
+        os.environ.update(update_after)
+        for k in remove_after:
+            os.environ.pop(k)
+
+
+class GetProjectSettingsTestCase(unittest.TestCase):
+
+    def test_valid_envvar(self):
+        value = 'tests.test_cmdline.settings'
+        envvars = {
+            'SCRAPY_SETTINGS_MODULE': value,
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            settings = get_project_settings()
+        assert not warnings
+        assert settings.get('SETTINGS_MODULE') == value
+
+    def test_invalid_envvar(self):
+        envvars = {
+            'SCRAPY_FOO': 'bar',
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            get_project_settings()
+        assert len(warnings) == 1
+        assert warnings[0].category == ScrapyDeprecationWarning
+        assert str(warnings[0].message).endswith(': FOO')
+
+    def test_valid_and_invalid_envvars(self):
+        value = 'tests.test_cmdline.settings'
+        envvars = {
+            'SCRAPY_FOO': 'bar',
+            'SCRAPY_SETTINGS_MODULE': value,
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            settings = get_project_settings()
+        assert len(warnings) == 1
+        assert warnings[0].category == ScrapyDeprecationWarning
+        assert str(warnings[0].message).endswith(': FOO')
+        assert settings.get('SETTINGS_MODULE') == value