From f001641543e110a44a5f384c19ba356c347d2e1e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 6 Jun 2015 04:05:54 +0500 Subject: [PATCH] (backwards-incompatible) allow to pass settings=None to configure_logging * use explicit argument for disabling root handler; * handle LOG_STDOUT even if install_root_handler is False (cherry picked from commit 9a787893e3dccd8bcbfa6821806fde2e2ef4515e) --- docs/topics/logging.rst | 23 +++++++++------ docs/topics/practices.rst | 4 +-- scrapy/utils/log.py | 59 ++++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 1a3f5d69f..469f0c81d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,7 +194,7 @@ scrapy.utils.log module .. module:: scrapy.utils.log :synopsis: Logging utils -.. function:: configure_logging(settings=None) +.. function:: configure_logging(settings=None, install_root_handler=True) This function initializes logging defaults for Scrapy. @@ -203,16 +203,17 @@ scrapy.utils.log module not required but it's recommended. This function does: - - Route warnings and Twisted logging through Python standard logging - - Set a filter on Scrapy logger for formatting Twisted failures - - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers - respectively + - Route warnings and twisted logging through Python standard logging + - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively + - Route stdout to log if LOG_STDOUT setting is True - If `settings` is not ``None``, it will also create a root handler based on - the settings listed in :ref:`topics-logging-settings`. + When ``install_root_handler`` is True (default), this function also + - Sets FailureFormatter filter on Scrapy logger + - Creates a root handler based on the settings listed + in :ref:`topics-logging-settings`. If you plan on configuring the handlers yourself is still recommended you - call this function, keeping `settings` as ``None``. Bear in mind there + call this function, passing `install_root_handler=False`. Bear in mind there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use @@ -222,7 +223,7 @@ scrapy.utils.log module import logging from scrapy.utils.log import configure_logging - configure_logging() # Note we aren't providing settings in this case + configure_logging(install_root_handler=False) logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO) Refer to :ref:`run-from-script` for more details about using Scrapy this @@ -232,4 +233,8 @@ scrapy.utils.log module root logger. :type settings: :class:`~scrapy.settings.Settings` object or ``None`` + :param install_root_handler: whether to install root logging handler + (default: True) + :type install_root_handler: bool + .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 4c8c83924..7666a84cf 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -148,7 +148,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() runner.crawl(MySpider1) runner.crawl(MySpider2) @@ -173,7 +173,7 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() @defer.inlineCallbacks diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 931e28f2d..0102d1740 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -56,14 +56,17 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None): +def configure_logging(settings=None, install_root_handler=True): """Initialize and configure default loggers This function does: - Route warnings and twisted logging through Python standard logging - - Set FailureFormatter filter on Scrapy logger - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Create a handler for the root logger according to given settings + - Route stdout to log if LOG_STDOUT setting is True + + When ``install_root_handler`` is True (default), this function also + - Sets FailureFormatter filter on Scrapy logger + - Creates a handler for the root logger according to given settings """ if not sys.warnoptions: # Route warnings through python logging @@ -74,35 +77,39 @@ def configure_logging(settings=None): dictConfig(DEFAULT_LOGGING) - if isinstance(settings, dict): + if isinstance(settings, dict) or settings is None: settings = Settings(settings) - if settings: + if settings.getbool('LOG_STDOUT'): + sys.stdout = StreamLogger(logging.getLogger('stdout')) + + if install_root_handler: logging.root.setLevel(logging.NOTSET) - - if settings.getbool('LOG_STDOUT'): - sys.stdout = StreamLogger(logging.getLogger('stdout')) - - # Set up the default log handler - filename = settings.get('LOG_FILE') - if filename: - encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) - elif settings.getbool('LOG_ENABLED'): - handler = logging.StreamHandler() - else: - handler = logging.NullHandler() - - formatter = logging.Formatter( - fmt=settings.get('LOG_FORMAT'), - datefmt=settings.get('LOG_DATEFORMAT') - ) - handler.setFormatter(formatter) - handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + handler = _get_handler(settings) logging.root.addHandler(handler) +def _get_handler(settings): + """ Return a log handler object according to settings """ + filename = settings.get('LOG_FILE') + if filename: + encoding = settings.get('LOG_ENCODING') + handler = logging.FileHandler(filename, encoding=encoding) + elif settings.getbool('LOG_ENABLED'): + handler = logging.StreamHandler() + else: + handler = logging.NullHandler() + + formatter = logging.Formatter( + fmt=settings.get('LOG_FORMAT'), + datefmt=settings.get('LOG_DATEFORMAT') + ) + handler.setFormatter(formatter) + handler.setLevel(settings.get('LOG_LEVEL')) + handler.addFilter(TopLevelFormatter(['scrapy'])) + return handler + + def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})