diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 57f2b6322..e8af90f11 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -10,7 +10,7 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, convert=update_classpath): +def build_component_list(compdict, custom=None, convert=update_classpath): """Compose a component list from a { class: order } dictionary.""" def _check_components(complist): @@ -34,9 +34,15 @@ def build_component_list(compdict, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in six.iteritems(compdict)} - if isinstance(compdict, (list, tuple)): - _check_components(compdict) - return type(compdict)(convert(c) for c in compdict) + # BEGIN Backwards compatibility for old (base, custom) call signature + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) + + if custom is not None: + compdict.update(custom) + # END Backwards compatibility + compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index af15d3184..dab41ac8d 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -8,46 +8,59 @@ class BuildComponentListTest(unittest.TestCase): def test_build_dict(self): d = {'one': 1, 'two': None, 'three': 8, 'four': 4} - self.assertEqual(build_component_list(d, lambda x: x), + self.assertEqual(build_component_list(d, convert=lambda x: x), ['one', 'four', 'three']) + def test_backwards_compatible_build_dict(self): + base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} + custom = {'two': None, 'three': 8, 'four': 4} + self.assertEqual(build_component_list(base, custom, + convert=lambda x: x), + ['one', 'four', 'five', 'three']) + def test_return_list(self): custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(custom, lambda x: x), custom) + self.assertEqual(build_component_list(None, custom, + convert=lambda x: x), + custom) def test_map_dict(self): custom = {'one': 1, 'two': 2, 'three': 3} - self.assertEqual(build_component_list(custom, lambda x: x.upper()), + self.assertEqual(build_component_list({}, custom, + convert=lambda x: x.upper()), ['ONE', 'TWO', 'THREE']) def test_map_list(self): custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(custom, lambda x: x.upper()), + self.assertEqual(build_component_list(None, custom, + lambda x: x.upper()), ['A', 'B', 'C']) def test_duplicate_components_in_dict(self): duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4} - self.assertRaises(ValueError, - build_component_list, duplicate_dict, lambda x: x.lower()) + self.assertRaises(ValueError, build_component_list, {}, duplicate_dict, + convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, - build_component_list, duplicate_list, lambda x: x) + self.assertRaises(ValueError, build_component_list, None, + duplicate_list, convert=lambda x: x) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0) duplicate_bs.set('ONE', 4, priority=10) - self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()), + self.assertEqual(build_component_list(duplicate_bs, + convert=lambda x: x.lower()), ['two', 'one']) duplicate_bs.set('one', duplicate_bs['one'], priority=20) - self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()), + self.assertEqual(build_component_list(duplicate_bs, + convert=lambda x: x.lower()), ['one', 'two']) # Same priority raises ValueError duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20) - self.assertRaises(ValueError, - build_component_list, duplicate_bs, convert=lambda x: x.lower()) + self.assertRaises(ValueError, build_component_list, duplicate_bs, + convert=lambda x: x.lower()) class UtilsConfTestCase(unittest.TestCase):