Replace BaseSettings._getcomposite() with public .getwithbase() method

scrapy/commands/check.py

@@ -58,7 +58,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         # load contracts
-        contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS'))
+        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
         conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)

scrapy/commands/crawl.py

@@ -35,7 +35,8 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(
+                self.settings.getwithbase('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")

scrapy/commands/runspider.py

@@ -58,7 +58,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")

scrapy/core/downloader/handlers/__init__.py

@@ -20,7 +20,8 @@ class DownloadHandlers(object):
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
-        handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
+        handlers = without_none_values(
+            crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
 

scrapy/core/downloader/middleware.py

@@ -19,7 +19,8 @@ class DownloaderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES'))
+        return build_component_list(
+            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         if hasattr(mw, 'process_request'):

scrapy/core/spidermw.py

@@ -18,7 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES'))
+        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         super(SpiderMiddlewareManager, self)._add_middleware(mw)

scrapy/extension.py

@@ -12,4 +12,4 @@ class ExtensionManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('EXTENSIONS'))
+        return build_component_list(settings.getwithbase('EXTENSIONS'))

scrapy/extensions/feedexport.py

@@ -196,7 +196,7 @@ class FeedExporter(object):
         return item
 
     def _load_components(self, setting_prefix):
-        conf = without_none_values(self.settings._getcomposite(setting_prefix))
+        conf = without_none_values(self.settings.getwithbase(setting_prefix))
         d = {}
         for k, v in conf.items():
             try:

scrapy/pipelines/__init__.py

@@ -13,7 +13,7 @@ class ItemPipelineManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('ITEM_PIPELINES'))
+        return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
 
     def _add_middleware(self, pipe):
         super(ItemPipelineManager, self)._add_middleware(pipe)

scrapy/settings/__init__.py

@@ -195,25 +195,17 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
-    def _getcomposite(self, name):
-        # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS
-        # It's for internal use in the transition away from the _BASE settings
-        # and will be removed along with _BASE support in a future release
-        basename = name + "_BASE"
-        if basename in self:
-            warnings.warn('_BASE settings are deprecated.',
-                          category=ScrapyDeprecationWarning)
-            # When users defined a _BASE setting, they explicitly don't want to
-            # use any of Scrapy's defaults. Therefore, we only use these entries
-            # from self[name] (where the defaults now live) that have a priority
-            # higher than 'default'
-            compsett = BaseSettings(self[basename], priority='default')
-            for k in self[name]:
-                prio = self[name].getpriority(k)
-                if prio > get_settings_priority('default'):
-                    compsett.set(k, self[name][k], prio)
-            return compsett
-        return self[name]
+    def getwithbase(self, name):
+        """Get a composition of a dictionary-like setting and its `_BASE`
+        counterpart.
+
+        :param name: name of the dictionary-like setting
+        :type name: string
+        """
+        compbs = BaseSettings()
+        compbs.update(self[name + '_BASE'])
+        compbs.update(self[name])
+        return compbs
 
     def getpriority(self, name):
         """

tests/test_settings/__init__.py

@@ -263,24 +263,15 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertEqual(settings.getpriority('key'), 99)
         self.assertEqual(settings.getpriority('nonexistentkey'), None)
 
-    def test_getcomposite(self):
-        s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
+    def test_getwithbase(self):
+        s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
-                          'HASNOBASE': BaseSettings({1: 1}, 'default')})
-        s['TEST'].set(4, 4, priority='project')
-        # When users specify a _BASE setting they explicitly don't want to use
-        # Scrapy's defaults, so we don't want to see anything that has a
-        # 'default' priority from TEST
-        cs = s._getcomposite('TEST')
-        self.assertEqual(len(cs), 3)
-        self.assertEqual(cs[1], 1)
-        self.assertEqual(cs[2], 2)
-        self.assertEqual(cs[4], 4)
-        cs = s._getcomposite('HASNOBASE')
-        self.assertEqual(len(cs), 1)
-        self.assertEqual(cs[1], 1)
-        cs = s._getcomposite('NONEXISTENT')
-        self.assertIsNone(cs)
+                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
+        s['TEST'].set(2, 200, 'cmdline')
+        six.assertCountEqual(self, s.getwithbase('TEST'),
+                             {1: 1, 2: 200, 3: 30})
+        six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+        self.assertEqual(s.getwithbase('NONEXISTENT'), {})
 
     def test_maxpriority(self):
         # Empty settings should return 'default'