Merge pull request #3660 from anubhavp28/logFormatter-doc-patch

[MRG+1] docs for scrapy.logformatter

docs/topics/logging.rst

@@ -193,6 +193,17 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
+.. _custom-log-formats:
+
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
+and making :setting:`LOG_FORMATTER` point to your new class.
+ 
+.. autoclass:: scrapy.logformatter.LogFormatter
+   :members:
+
 Advanced customization
 ----------------------
 

docs/topics/settings.rst

@@ -870,6 +870,15 @@ directives.
 
 .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
 
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: :class:`scrapy.logformatter.LogFormatter`
+
+The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
+
 .. setting:: LOG_LEVEL
 
 LOG_LEVEL

scrapy/logformatter.py

@@ -12,26 +12,40 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref
 
 class LogFormatter(object):
     """Class for generating log messages for different actions.
-
-    All methods must return a dictionary listing the parameters ``level``,
-    ``msg`` and ``args`` which are going to be used for constructing the log
-    message when calling logging.log.
+    
+    All methods must return a dictionary listing the parameters ``level``, ``msg``
+    and ``args`` which are going to be used for constructing the log message when
+    calling ``logging.log``.
 
     Dictionary keys for the method outputs:
-        * ``level`` should be the log level for that action, you can use those
-        from the python logging library: logging.DEBUG, logging.INFO,
-        logging.WARNING, logging.ERROR and logging.CRITICAL.
 
-        * ``msg`` should be a string that can contain different formatting
-        placeholders. This string, formatted with the provided ``args``, is
-        going to be the log message for that action.
+    *   ``level`` is the log level for that action, you can use those from the
+        `python logging library <https://docs.python.org/3/library/logging.html>`_ :
+        ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
+        and ``logging.CRITICAL``.
+    *   ``msg`` should be a string that can contain different formatting placeholders.
+        This string, formatted with the provided ``args``, is going to be the long message
+        for that action.
+    *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
+        The final log message is computed as ``msg % args``.
 
-        * ``args`` should be a tuple or dict with the formatting placeholders
-        for ``msg``.  The final log message is computed as output['msg'] %
-        output['args'].
+    Here is an example on how to create a custom log formatter to lower the severity level of
+    the log message when an item is dropped from the pipeline::
+
+            class PoliteLogFormatter(logformatter.LogFormatter):
+                def dropped(self, item, exception, response, spider):
+                    return {
+                        'level': logging.INFO, # lowering the level from logging.WARNING
+                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'args': {
+                            'exception': exception,
+                            'item': item,
+                        }
+                    }
     """
-
+    
     def crawled(self, request, response, spider):
+        """Logs a message when the crawler finds a webpage."""
         request_flags = ' %s' % str(request.flags) if request.flags else ''
         response_flags = ' %s' % str(response.flags) if response.flags else ''
         return {
@@ -40,7 +54,7 @@ class LogFormatter(object):
             'args': {
                 'status': response.status,
                 'request': request,
-                'request_flags' : request_flags,
+                'request_flags': request_flags,
                 'referer': referer_str(request),
                 'response_flags': response_flags,
                 # backward compatibility with Scrapy logformatter below 1.4 version
@@ -49,6 +63,7 @@ class LogFormatter(object):
         }
 
     def scraped(self, item, response, spider):
+        """Logs a message when an item is scraped by a spider."""
         if isinstance(response, Failure):
             src = response.getErrorMessage()
         else:
@@ -63,6 +78,7 @@ class LogFormatter(object):
         }
 
     def dropped(self, item, exception, response, spider):
+        """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
             'level': logging.WARNING,
             'msg': DROPPEDMSG,