From 3823b032ffe287f461ae7d6c3ff8e9a02aa39433 Mon Sep 17 00:00:00 2001 From: Taito Horiuchi Date: Thu, 15 Sep 2016 09:43:19 +0300 Subject: [PATCH 1/4] Add LOG_HANDLER to settings, so that other file handlers can be chosen --- scrapy/utils/log.py | 15 ++++++++++++++- tests/test_utils_log_handler.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/test_utils_log_handler.py diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cc2f0b164..d3fde097c 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -101,12 +101,25 @@ def configure_logging(settings=None, install_root_handler=True): logging.root.addHandler(handler) +def _import_hander(name): + """Return class from dotted name""" + import importlib + module_name, class_name = name.rsplit('.', 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) + handler = settings.get('LOG_HANDLER') + if handler: + Handler = _import_hander(handler) + else: + Handler = logging.FileHandler + handler = Handler(filename, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: diff --git a/tests/test_utils_log_handler.py b/tests/test_utils_log_handler.py new file mode 100644 index 000000000..fecb70630 --- /dev/null +++ b/tests/test_utils_log_handler.py @@ -0,0 +1,29 @@ +from logging import FileHandler +from logging.handlers import TimedRotatingFileHandler +from scrapy.utils import log +import unittest + + +class TestCase(unittest.TestCase): + + + def test_import_handler(self): + """Test function: _import_handler""" + name = 'logging.handlers.TimedRotatingFileHandler' + self.assertEqual(log._import_hander(name), TimedRotatingFileHandler) + + def test_get_handler(self): + """Test function: _get_handler with LOG_FILE""" + # First create log file path + import os + import tempfile + file_path = tempfile.mkstemp()[1] + settings = {'LOG_FILE': file_path, 'LOG_ENCODIUNG': 'utf-8', 'LOG_LEVEL': 'DEBUG'} + handler = log._get_handler(settings) + self.assertIsInstance(handler, FileHandler) + # Adding LOG_HANDLER to settings should update the handler + settings['LOG_HANDLER'] = 'logging.handlers.TimedRotatingFileHandler' + handler = log._get_handler(settings) + self.assertIsInstance(handler, TimedRotatingFileHandler) + # Remove log file + os.remove(file_path) From c4939f464bf82450ddcf5942e057c90f1d030018 Mon Sep 17 00:00:00 2001 From: Taito Horiuchi Date: Fri, 13 Jan 2017 13:28:14 +0200 Subject: [PATCH 2/4] enable settings.LOGGING to benefit dictConfig --- scrapy/utils/log.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0ea91872b..09a7502be 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -92,34 +92,29 @@ def configure_logging(settings=None, install_root_handler=True): if isinstance(settings, dict) or settings is None: settings = Settings(settings) + LOGGING_FROM_SETTINGS = settings.get('LOGGING') + if settings.getbool('LOG_STDOUT'): sys.stdout = StreamLogger(logging.getLogger('stdout')) + LOGGING_FROM_SETTINGS = None - if install_root_handler: + elif LOGGING_FROM_SETTINGS: + LOGGING = DEFAULT_LOGGING.copy() + LOGGING.update(LOGGING_FROM_SETTINGS) + dictConfig(LOGGING) + + if install_root_handler and not LOGGING_FROM_SETTINGS: logging.root.setLevel(logging.NOTSET) handler = _get_handler(settings) logging.root.addHandler(handler) -def _import_hander(name): - """Return class from dotted name""" - import importlib - module_name, class_name = name.rsplit('.', 1) - module = importlib.import_module(module_name) - return getattr(module, class_name) - - def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: encoding = settings.get('LOG_ENCODING') - handler = settings.get('LOG_HANDLER') - if handler: - Handler = _import_hander(handler) - else: - Handler = logging.FileHandler - handler = Handler(filename, encoding=encoding) + handler = logging.FileHandler(filename, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: From e614fbbb108247d7dc8ab31603b177a1078e5a27 Mon Sep 17 00:00:00 2001 From: Taito Horiuchi Date: Fri, 13 Jan 2017 14:11:52 +0200 Subject: [PATCH 3/4] use settings.getdict instead of settings.get test configure_logging --- scrapy/utils/log.py | 2 +- tests/test_utils_log_handler.py | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 09a7502be..6f67f7989 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -92,7 +92,7 @@ def configure_logging(settings=None, install_root_handler=True): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - LOGGING_FROM_SETTINGS = settings.get('LOGGING') + LOGGING_FROM_SETTINGS = settings.getdict('LOGGING') if settings.getbool('LOG_STDOUT'): sys.stdout = StreamLogger(logging.getLogger('stdout')) diff --git a/tests/test_utils_log_handler.py b/tests/test_utils_log_handler.py index fecb70630..5d6bb0697 100644 --- a/tests/test_utils_log_handler.py +++ b/tests/test_utils_log_handler.py @@ -1,29 +1,18 @@ -from logging import FileHandler -from logging.handlers import TimedRotatingFileHandler +# from logging import FileHandler +# from logging.handlers import TimedRotatingFileHandler from scrapy.utils import log +import logging import unittest class TestCase(unittest.TestCase): + def test_settings_None(self): + log.configure_logging() + self.assertEqual(logging.getLogger().getEffectiveLevel(), 0) - def test_import_handler(self): - """Test function: _import_handler""" - name = 'logging.handlers.TimedRotatingFileHandler' - self.assertEqual(log._import_hander(name), TimedRotatingFileHandler) - - def test_get_handler(self): - """Test function: _get_handler with LOG_FILE""" - # First create log file path - import os - import tempfile - file_path = tempfile.mkstemp()[1] - settings = {'LOG_FILE': file_path, 'LOG_ENCODIUNG': 'utf-8', 'LOG_LEVEL': 'DEBUG'} - handler = log._get_handler(settings) - self.assertIsInstance(handler, FileHandler) - # Adding LOG_HANDLER to settings should update the handler - settings['LOG_HANDLER'] = 'logging.handlers.TimedRotatingFileHandler' - handler = log._get_handler(settings) - self.assertIsInstance(handler, TimedRotatingFileHandler) - # Remove log file - os.remove(file_path) + def test_settings_LOGGING(self): + settings = {'LOGGING': {'loggers': {'logger3': {'level': 'CRITICAL'}}}} + log.configure_logging(settings=settings) + self.assertEqual(logging.getLogger().getEffectiveLevel(), 30) + self.assertEqual(logging.getLogger('logger3').getEffectiveLevel(), 50) From 0980e794a9f43b8497aae9d1ce63cdf931ad997c Mon Sep 17 00:00:00 2001 From: Taito Horiuchi Date: Tue, 28 Feb 2017 15:08:41 +0200 Subject: [PATCH 4/4] refactor --- scrapy/utils/log.py | 11 +++++------ tests/test_utils_log_handler.py | 2 -- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6f67f7989..6954c5009 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -92,18 +92,17 @@ def configure_logging(settings=None, install_root_handler=True): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - LOGGING_FROM_SETTINGS = settings.getdict('LOGGING') - if settings.getbool('LOG_STDOUT'): sys.stdout = StreamLogger(logging.getLogger('stdout')) - LOGGING_FROM_SETTINGS = None - elif LOGGING_FROM_SETTINGS: + logging_from_settings = settings.getdict('LOGGING') + + if logging_from_settings: LOGGING = DEFAULT_LOGGING.copy() - LOGGING.update(LOGGING_FROM_SETTINGS) + LOGGING.update(logging_from_settings) dictConfig(LOGGING) - if install_root_handler and not LOGGING_FROM_SETTINGS: + elif install_root_handler: logging.root.setLevel(logging.NOTSET) handler = _get_handler(settings) logging.root.addHandler(handler) diff --git a/tests/test_utils_log_handler.py b/tests/test_utils_log_handler.py index 5d6bb0697..ccf15f44a 100644 --- a/tests/test_utils_log_handler.py +++ b/tests/test_utils_log_handler.py @@ -1,5 +1,3 @@ -# from logging import FileHandler -# from logging.handlers import TimedRotatingFileHandler from scrapy.utils import log import logging import unittest