diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dea0528db..87ea43c7d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,6 +193,17 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +.. _custom-log-formats: + +Custom Log Formats +------------------ + +A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` point to your new class. + +.. autoclass:: scrapy.logformatter.LogFormatter + :members: + Advanced customization ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fd46c614e..617f1dcdf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -870,6 +870,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: :class:`scrapy.logformatter.LogFormatter` + +The class to use for :ref:`formatting log messages ` for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 65f347dcf..b4d6787ff 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -12,26 +12,40 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - - All methods must return a dictionary listing the parameters ``level``, - ``msg`` and ``args`` which are going to be used for constructing the log - message when calling logging.log. + + All methods must return a dictionary listing the parameters ``level``, ``msg`` + and ``args`` which are going to be used for constructing the log message when + calling ``logging.log``. Dictionary keys for the method outputs: - * ``level`` should be the log level for that action, you can use those - from the python logging library: logging.DEBUG, logging.INFO, - logging.WARNING, logging.ERROR and logging.CRITICAL. - * ``msg`` should be a string that can contain different formatting - placeholders. This string, formatted with the provided ``args``, is - going to be the log message for that action. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + * ``msg`` should be a string that can contain different formatting placeholders. + This string, formatted with the provided ``args``, is going to be the long message + for that action. + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. + The final log message is computed as ``msg % args``. - * ``args`` should be a tuple or dict with the formatting placeholders - for ``msg``. The final log message is computed as output['msg'] % - output['args']. + Here is an example on how to create a custom log formatter to lower the severity level of + the log message when an item is dropped from the pipeline:: + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + 'level': logging.INFO, # lowering the level from logging.WARNING + 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'args': { + 'exception': exception, + 'item': item, + } + } """ - + def crawled(self, request, response, spider): + """Logs a message when the crawler finds a webpage.""" request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -40,7 +54,7 @@ class LogFormatter(object): 'args': { 'status': response.status, 'request': request, - 'request_flags' : request_flags, + 'request_flags': request_flags, 'referer': referer_str(request), 'response_flags': response_flags, # backward compatibility with Scrapy logformatter below 1.4 version @@ -49,6 +63,7 @@ class LogFormatter(object): } def scraped(self, item, response, spider): + """Logs a message when an item is scraped by a spider.""" if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -63,6 +78,7 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): + """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { 'level': logging.WARNING, 'msg': DROPPEDMSG,