diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 1a3f5d69f..062c6e518 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,42 +194,33 @@ scrapy.utils.log module .. module:: scrapy.utils.log :synopsis: Logging utils -.. function:: configure_logging(settings=None) +.. autofunction:: configure_logging - This function initializes logging defaults for Scrapy. - - It's automatically called when using Scrapy commands, but needs to be - called explicitely when running custom scripts. In that case, its usage is - not required but it's recommended. - - This function does: - - Route warnings and Twisted logging through Python standard logging - - Set a filter on Scrapy logger for formatting Twisted failures - - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers - respectively - - If `settings` is not ``None``, it will also create a root handler based on - the settings listed in :ref:`topics-logging-settings`. + ``configure_logging`` is automatically called when using Scrapy commands, + but needs to be called explicitly when running custom scripts. In that + case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you - call this function, keeping `settings` as ``None``. Bear in mind there - won't be any log output set by default in that case. + call this function, passing `install_root_handler=False`. Bear in mind + there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use - `logging.basicConfig()`_ to set a basic root handler. This is an example on - how to redirect ``INFO`` or higher messages to a file:: + `logging.basicConfig()`_ to set a basic root handler. This is an example + on how to redirect ``INFO`` or higher messages to a file:: import logging from scrapy.utils.log import configure_logging - configure_logging() # Note we aren't providing settings in this case - logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO) + configure_logging(install_root_handler=False) + logging.basicConfig( + filename='log.txt', + format='%(levelname)s: %(message)s', + level=logging.INFO + ) Refer to :ref:`run-from-script` for more details about using Scrapy this way. - :param settings: settings used to create and configure a handler for the - root logger. - :type settings: :class:`~scrapy.settings.Settings` object or ``None`` - .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig + + diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 4c8c83924..7666a84cf 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -148,7 +148,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() runner.crawl(MySpider1) runner.crawl(MySpider2) @@ -173,7 +173,7 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging({}) + configure_logging() runner = CrawlerRunner() @defer.inlineCallbacks diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 931e28f2d..d40202953 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -56,14 +56,29 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None): - """Initialize and configure default loggers +def configure_logging(settings=None, install_root_handler=True): + """ + Initialize logging defaults for Scrapy. + + :param settings: settings used to create and configure a handler for the + root logger (default: None). + :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None`` + + :param install_root_handler: whether to install root logging handler + (default: True) + :type install_root_handler: bool This function does: - - Route warnings and twisted logging through Python standard logging - - Set FailureFormatter filter on Scrapy logger - - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Create a handler for the root logger according to given settings + + - Route warnings and twisted logging through Python standard logging + - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively + - Route stdout to log if LOG_STDOUT setting is True + + When ``install_root_handler`` is True (default), this function also + creates a handler for the root logger according to given settings + (see :ref:`topics-logging-settings`). You can override default options + using ``settings`` argument. When ``settings`` is empty or None, defaults + are used. """ if not sys.warnoptions: # Route warnings through python logging @@ -74,35 +89,39 @@ def configure_logging(settings=None): dictConfig(DEFAULT_LOGGING) - if isinstance(settings, dict): + if isinstance(settings, dict) or settings is None: settings = Settings(settings) - if settings: + if settings.getbool('LOG_STDOUT'): + sys.stdout = StreamLogger(logging.getLogger('stdout')) + + if install_root_handler: logging.root.setLevel(logging.NOTSET) - - if settings.getbool('LOG_STDOUT'): - sys.stdout = StreamLogger(logging.getLogger('stdout')) - - # Set up the default log handler - filename = settings.get('LOG_FILE') - if filename: - encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) - elif settings.getbool('LOG_ENABLED'): - handler = logging.StreamHandler() - else: - handler = logging.NullHandler() - - formatter = logging.Formatter( - fmt=settings.get('LOG_FORMAT'), - datefmt=settings.get('LOG_DATEFORMAT') - ) - handler.setFormatter(formatter) - handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + handler = _get_handler(settings) logging.root.addHandler(handler) +def _get_handler(settings): + """ Return a log handler object according to settings """ + filename = settings.get('LOG_FILE') + if filename: + encoding = settings.get('LOG_ENCODING') + handler = logging.FileHandler(filename, encoding=encoding) + elif settings.getbool('LOG_ENABLED'): + handler = logging.StreamHandler() + else: + handler = logging.NullHandler() + + formatter = logging.Formatter( + fmt=settings.get('LOG_FORMAT'), + datefmt=settings.get('LOG_DATEFORMAT') + ) + handler.setFormatter(formatter) + handler.setLevel(settings.get('LOG_LEVEL')) + handler.addFilter(TopLevelFormatter(['scrapy'])) + return handler + + def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})