From 9a787893e3dccd8bcbfa6821806fde2e2ef4515e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 6 Jun 2015 04:05:54 +0500 Subject: [PATCH 1/3] (backwards-incompatible) allow to pass settings=None to configure_logging * use explicit argument for disabling root handler; * handle LOG_STDOUT even if install_root_handler is False --- docs/topics/logging.rst | 23 +++++++++------ docs/topics/practices.rst | 4 +-- scrapy/utils/log.py | 59 ++++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 1a3f5d69f..469f0c81d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,7 +194,7 @@ scrapy.utils.log module .. module:: scrapy.utils.log :synopsis: Logging utils -.. function:: configure_logging(settings=None) +.. function:: configure_logging(settings=None, install_root_handler=True) This function initializes logging defaults for Scrapy. @@ -203,16 +203,17 @@ scrapy.utils.log module not required but it's recommended. This function does: - - Route warnings and Twisted logging through Python standard logging - - Set a filter on Scrapy logger for formatting Twisted failures - - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers - respectively + - Route warnings and twisted logging through Python standard logging + - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively + - Route stdout to log if LOG_STDOUT setting is True - If `settings` is not ``None``, it will also create a root handler based on - the settings listed in :ref:`topics-logging-settings`. + When ``install_root_handler`` is True (default), this function also + - Sets FailureFormatter filter on Scrapy logger + - Creates a root handler based on the settings listed + in :ref:`topics-logging-settings`. If you plan on configuring the handlers yourself is still recommended you - call this function, keeping `settings` as ``None``. Bear in mind there + call this function, passing `install_root_handler=False`. Bear in mind there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use @@ -222,7 +223,7 @@ scrapy.utils.log module import logging from scrapy.utils.log import configure_logging - configure_logging() # Note we aren't providing settings in this case + configure_logging(install_root_handler=False) logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO) Refer to :ref:`run-from-script` for more details about using Scrapy this @@ -232,4 +233,8 @@ scrapy.utils.log module root logger. :type settings: :class:`~scrapy.settings.Settings` object or ``None`` + :param install_root_handler: whether to install root logging handler + (default: True) + :type install_root_handler: bool + .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 4c8c83924..7666a84cf 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -148,7 +148,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() runner.crawl(MySpider1) runner.crawl(MySpider2) @@ -173,7 +173,7 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() @defer.inlineCallbacks diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 931e28f2d..0102d1740 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -56,14 +56,17 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None): +def configure_logging(settings=None, install_root_handler=True): """Initialize and configure default loggers This function does: - Route warnings and twisted logging through Python standard logging - - Set FailureFormatter filter on Scrapy logger - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Create a handler for the root logger according to given settings + - Route stdout to log if LOG_STDOUT setting is True + + When ``install_root_handler`` is True (default), this function also + - Sets FailureFormatter filter on Scrapy logger + - Creates a handler for the root logger according to given settings """ if not sys.warnoptions: # Route warnings through python logging @@ -74,35 +77,39 @@ def configure_logging(settings=None): dictConfig(DEFAULT_LOGGING) - if isinstance(settings, dict): + if isinstance(settings, dict) or settings is None: settings = Settings(settings) - if settings: + if settings.getbool('LOG_STDOUT'): + sys.stdout = StreamLogger(logging.getLogger('stdout')) + + if install_root_handler: logging.root.setLevel(logging.NOTSET) - - if settings.getbool('LOG_STDOUT'): - sys.stdout = StreamLogger(logging.getLogger('stdout')) - - # Set up the default log handler - filename = settings.get('LOG_FILE') - if filename: - encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) - elif settings.getbool('LOG_ENABLED'): - handler = logging.StreamHandler() - else: - handler = logging.NullHandler() - - formatter = logging.Formatter( - fmt=settings.get('LOG_FORMAT'), - datefmt=settings.get('LOG_DATEFORMAT') - ) - handler.setFormatter(formatter) - handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + handler = _get_handler(settings) logging.root.addHandler(handler) +def _get_handler(settings): + """ Return a log handler object according to settings """ + filename = settings.get('LOG_FILE') + if filename: + encoding = settings.get('LOG_ENCODING') + handler = logging.FileHandler(filename, encoding=encoding) + elif settings.getbool('LOG_ENABLED'): + handler = logging.StreamHandler() + else: + handler = logging.NullHandler() + + formatter = logging.Formatter( + fmt=settings.get('LOG_FORMAT'), + datefmt=settings.get('LOG_DATEFORMAT') + ) + handler.setFormatter(formatter) + handler.setLevel(settings.get('LOG_LEVEL')) + handler.addFilter(TopLevelFormatter(['scrapy'])) + return handler + + def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) From a611f8dd2dd9fb9f7fe81245a488c4493744014d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jun 2015 22:57:18 +0500 Subject: [PATCH 2/3] DOC remove FailureFormatter mentions, stop copy-pasting configure_logging docstring --- docs/topics/logging.rst | 58 ++++++++++++++++------------------------- scrapy/utils/log.py | 24 ++++++++++++----- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 469f0c81d..3d7ce958c 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,47 +194,33 @@ scrapy.utils.log module .. module:: scrapy.utils.log :synopsis: Logging utils -.. function:: configure_logging(settings=None, install_root_handler=True) +.. autofunction:: configure_logging - This function initializes logging defaults for Scrapy. +``configure_logging`` is automatically called when using Scrapy commands, +but needs to be called explicitly when running custom scripts. In that +case, its usage is not required but it's recommended. - It's automatically called when using Scrapy commands, but needs to be - called explicitely when running custom scripts. In that case, its usage is - not required but it's recommended. +If you plan on configuring the handlers yourself is still recommended you +call this function, passing `install_root_handler=False`. Bear in mind +there won't be any log output set by default in that case. - This function does: - - Route warnings and twisted logging through Python standard logging - - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Route stdout to log if LOG_STDOUT setting is True +To get you started on manually configuring logging's output, you can use +`logging.basicConfig()`_ to set a basic root handler. This is an example on +how to redirect ``INFO`` or higher messages to a file:: - When ``install_root_handler`` is True (default), this function also - - Sets FailureFormatter filter on Scrapy logger - - Creates a root handler based on the settings listed - in :ref:`topics-logging-settings`. + import logging + from scrapy.utils.log import configure_logging - If you plan on configuring the handlers yourself is still recommended you - call this function, passing `install_root_handler=False`. Bear in mind there - won't be any log output set by default in that case. + configure_logging(install_root_handler=False) + logging.basicConfig( + filename='log.txt', + format='%(levelname)s: %(message)s', + level=logging.INFO + ) - To get you started on manually configuring logging's output, you can use - `logging.basicConfig()`_ to set a basic root handler. This is an example on - how to redirect ``INFO`` or higher messages to a file:: - - import logging - from scrapy.utils.log import configure_logging - - configure_logging(install_root_handler=False) - logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO) - - Refer to :ref:`run-from-script` for more details about using Scrapy this - way. - - :param settings: settings used to create and configure a handler for the - root logger. - :type settings: :class:`~scrapy.settings.Settings` object or ``None`` - - :param install_root_handler: whether to install root logging handler - (default: True) - :type install_root_handler: bool +Refer to :ref:`run-from-script` for more details about using Scrapy this +way. .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig + + diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0102d1740..d40202953 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -57,16 +57,28 @@ DEFAULT_LOGGING = { def configure_logging(settings=None, install_root_handler=True): - """Initialize and configure default loggers + """ + Initialize logging defaults for Scrapy. + + :param settings: settings used to create and configure a handler for the + root logger (default: None). + :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None`` + + :param install_root_handler: whether to install root logging handler + (default: True) + :type install_root_handler: bool This function does: - - Route warnings and twisted logging through Python standard logging - - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Route stdout to log if LOG_STDOUT setting is True + + - Route warnings and twisted logging through Python standard logging + - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively + - Route stdout to log if LOG_STDOUT setting is True When ``install_root_handler`` is True (default), this function also - - Sets FailureFormatter filter on Scrapy logger - - Creates a handler for the root logger according to given settings + creates a handler for the root logger according to given settings + (see :ref:`topics-logging-settings`). You can override default options + using ``settings`` argument. When ``settings`` is empty or None, defaults + are used. """ if not sys.warnoptions: # Route warnings through python logging From 36bc912cdd061f6fd7845254133e582c33cbab3d Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 12 Jun 2015 12:56:57 -0300 Subject: [PATCH 3/3] DOC indent additional docs for configure_logging --- docs/topics/logging.rst | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 3d7ce958c..062c6e518 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -196,30 +196,30 @@ scrapy.utils.log module .. autofunction:: configure_logging -``configure_logging`` is automatically called when using Scrapy commands, -but needs to be called explicitly when running custom scripts. In that -case, its usage is not required but it's recommended. + ``configure_logging`` is automatically called when using Scrapy commands, + but needs to be called explicitly when running custom scripts. In that + case, its usage is not required but it's recommended. -If you plan on configuring the handlers yourself is still recommended you -call this function, passing `install_root_handler=False`. Bear in mind -there won't be any log output set by default in that case. + If you plan on configuring the handlers yourself is still recommended you + call this function, passing `install_root_handler=False`. Bear in mind + there won't be any log output set by default in that case. -To get you started on manually configuring logging's output, you can use -`logging.basicConfig()`_ to set a basic root handler. This is an example on -how to redirect ``INFO`` or higher messages to a file:: + To get you started on manually configuring logging's output, you can use + `logging.basicConfig()`_ to set a basic root handler. This is an example + on how to redirect ``INFO`` or higher messages to a file:: - import logging - from scrapy.utils.log import configure_logging + import logging + from scrapy.utils.log import configure_logging - configure_logging(install_root_handler=False) - logging.basicConfig( - filename='log.txt', - format='%(levelname)s: %(message)s', - level=logging.INFO - ) + configure_logging(install_root_handler=False) + logging.basicConfig( + filename='log.txt', + format='%(levelname)s: %(message)s', + level=logging.INFO + ) -Refer to :ref:`run-from-script` for more details about using Scrapy this -way. + Refer to :ref:`run-from-script` for more details about using Scrapy this + way. .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig