From a1ebff83d39e65cf4ed34281a1e81ea6cd108fe0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:39:34 +0100 Subject: [PATCH 1/4] Remove __str__ and __repr__ from settings, introduce copy_to_dict() instead Settings instances as dict's are easier to print or pretty print in the shell Fixes #1732 --- scrapy/settings/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..918bfc1e5 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -368,11 +368,25 @@ class BaseSettings(MutableMapping): def __len__(self): return len(self.attributes) - def __str__(self): - return str(self.attributes) + def _to_dict(self): + return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + for k, v in six.iteritems(self)} - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.attributes) + def copy_to_dict(self): + """ + Make a copy of current settings and convert to a dict. + + This method returns a new dict populated with the same values + and their priorities as the current settings. + + Modifications to the returned dict won't be reflected on the original + settings. + + This method can be useful for example for printing settings + in Scrapy shell. + """ + settings = self.copy() + return settings._to_dict() @property def overrides(self): From aa78758bc744b3264c330cc18e58a8b15315517d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:59:12 +0100 Subject: [PATCH 2/4] Update tests for settings copy_to_dict() --- tests/test_settings/__init__.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 44b9b6df3..4acf22cba 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -302,6 +302,21 @@ class BaseSettingsTest(unittest.TestCase): self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], ['first_one', 'first_two']) + def test_copy_to_dict(self): + s = BaseSettings({'TEST_STRING': 'a string', + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), + 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), + 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + self.assertDictEqual(s.copy_to_dict(), + {'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_BOOLEAN': False, + 'TEST_LIST': [1, 2], + 'TEST_STRING': 'a string'}) + def test_freeze(self): self.settings.freeze() with self.assertRaises(TypeError) as cm: @@ -343,14 +358,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(self.settings.defaults.get('BAR'), 'foo') self.assertIn('BAR', self.settings.defaults) - def test_repr(self): - settings = BaseSettings() - self.assertEqual(repr(settings), "") - attr = SettingsAttribute('testval', 15) - settings['testkey'] = attr - self.assertEqual(repr(settings), - "" % repr(attr)) - class SettingsTest(unittest.TestCase): From d843a0aae862a84b54f18f9d43ae957c182197b6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 21:12:03 +0100 Subject: [PATCH 3/4] Amend "settings" command to output JSON for dict settings --- scrapy/commands/settings.py | 9 ++++++++- tests/test_cmdline/__init__.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 0e73f4f58..bce4e6086 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,8 @@ from __future__ import print_function +import json + from scrapy.commands import ScrapyCommand +from scrapy.settings import BaseSettings class Command(ScrapyCommand): @@ -28,7 +31,11 @@ class Command(ScrapyCommand): def run(self, args, opts): settings = self.crawler_process.settings if opts.get: - print(settings.get(opts.get)) + s = settings.get(opts.get) + if isinstance(s, BaseSettings): + print(json.dumps(s.copy_to_dict())) + else: + print(s) elif opts.getbool: print(settings.getbool(opts.getbool)) elif opts.getint: diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index c2de4fbc8..7733e7180 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertIn('value=200', settingsdict[EXT_PATH]) + self.assertEquals(200, settingsdict[EXT_PATH]) From 268e912273dcb7bacc5a7102d5a3f90a868f035f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 1 Feb 2016 12:43:27 +0100 Subject: [PATCH 4/4] Add pretty-printting of settings as dict if using IPython shell Suggested by @digenis see http://ipython.readthedocs.org/en/stable/api/generated/IPython.lib.pretty.html?#extending --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 918bfc1e5..7b7808959 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,6 +4,7 @@ import copy import warnings from collections import MutableMapping from importlib import import_module +from pprint import pformat from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning @@ -388,6 +389,12 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() + def _repr_pretty_(self, p, cycle): + if cycle: + p.text(repr(self)) + else: + p.text(pformat(self.copy_to_dict())) + @property def overrides(self): warnings.warn("`Settings.overrides` attribute is deprecated and won't "