From 52ecee6a6271ef2d981f7f1993311abd8ed31bd9 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 13:15:32 +0100 Subject: [PATCH] Replace BaseSettings._getcomposite() with public .getwithbase() method --- scrapy/commands/check.py | 2 +- scrapy/commands/crawl.py | 3 ++- scrapy/commands/runspider.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 3 ++- scrapy/core/downloader/middleware.py | 3 ++- scrapy/core/spidermw.py | 2 +- scrapy/extension.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/__init__.py | 2 +- scrapy/settings/__init__.py | 30 ++++++++------------- tests/test_settings/__init__.py | 25 ++++++----------- 11 files changed, 31 insertions(+), 45 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index a423ba2c9..b8a9ef989 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -58,7 +58,7 @@ class Command(ScrapyCommand): def run(self, args, opts): # load contracts - contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS')) + contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS')) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 7f5c64c20..4b986bf9d 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -35,7 +35,8 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values( + self.settings.getwithbase('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 72229bcf5..1da09e4da 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -58,7 +58,7 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 0e78e04f4..bc5cd742e 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -20,7 +20,8 @@ class DownloadHandlers(object): self._schemes = {} # stores acceptable schemes on instancing self._handlers = {} # stores instanced handlers for schemes self._notconfigured = {} # remembers failed handlers - handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS')) + handlers = without_none_values( + crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 958113fc3..c3b23e284 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -19,7 +19,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES')) + return build_component_list( + settings.getwithbase('DOWNLOADER_MIDDLEWARES')) def _add_middleware(self, mw): if hasattr(mw, 'process_request'): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b5c80c350..a206e4b0c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -18,7 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES')) + return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): super(SpiderMiddlewareManager, self)._add_middleware(mw) diff --git a/scrapy/extension.py b/scrapy/extension.py index 4ceb32c68..e39e456fa 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -12,4 +12,4 @@ class ExtensionManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('EXTENSIONS')) + return build_component_list(settings.getwithbase('EXTENSIONS')) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1e27a1e7e..daea551cb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -196,7 +196,7 @@ class FeedExporter(object): return item def _load_components(self, setting_prefix): - conf = without_none_values(self.settings._getcomposite(setting_prefix)) + conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): try: diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 8df0d3154..2ef8786d0 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -13,7 +13,7 @@ class ItemPipelineManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('ITEM_PIPELINES')) + return build_component_list(settings.getwithbase('ITEM_PIPELINES')) def _add_middleware(self, pipe): super(ItemPipelineManager, self)._add_middleware(pipe) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 13656298b..b0f59ccc7 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -195,25 +195,17 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def _getcomposite(self, name): - # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS - # It's for internal use in the transition away from the _BASE settings - # and will be removed along with _BASE support in a future release - basename = name + "_BASE" - if basename in self: - warnings.warn('_BASE settings are deprecated.', - category=ScrapyDeprecationWarning) - # When users defined a _BASE setting, they explicitly don't want to - # use any of Scrapy's defaults. Therefore, we only use these entries - # from self[name] (where the defaults now live) that have a priority - # higher than 'default' - compsett = BaseSettings(self[basename], priority='default') - for k in self[name]: - prio = self[name].getpriority(k) - if prio > get_settings_priority('default'): - compsett.set(k, self[name][k], prio) - return compsett - return self[name] + def getwithbase(self, name): + """Get a composition of a dictionary-like setting and its `_BASE` + counterpart. + + :param name: name of the dictionary-like setting + :type name: string + """ + compbs = BaseSettings() + compbs.update(self[name + '_BASE']) + compbs.update(self[name]) + return compbs def getpriority(self, name): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4ef08bb0b..8d98d2cec 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -263,24 +263,15 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getpriority('key'), 99) self.assertEqual(settings.getpriority('nonexistentkey'), None) - def test_getcomposite(self): - s = BaseSettings({'TEST_BASE': {1: 1, 2: 2}, + def test_getwithbase(self): + s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), - 'HASNOBASE': BaseSettings({1: 1}, 'default')}) - s['TEST'].set(4, 4, priority='project') - # When users specify a _BASE setting they explicitly don't want to use - # Scrapy's defaults, so we don't want to see anything that has a - # 'default' priority from TEST - cs = s._getcomposite('TEST') - self.assertEqual(len(cs), 3) - self.assertEqual(cs[1], 1) - self.assertEqual(cs[2], 2) - self.assertEqual(cs[4], 4) - cs = s._getcomposite('HASNOBASE') - self.assertEqual(len(cs), 1) - self.assertEqual(cs[1], 1) - cs = s._getcomposite('NONEXISTENT') - self.assertIsNone(cs) + 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + s['TEST'].set(2, 200, 'cmdline') + six.assertCountEqual(self, s.getwithbase('TEST'), + {1: 1, 2: 200, 3: 30}) + six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): # Empty settings should return 'default'