From 82d239f3b148d9ce69f67bd7a2cb00de7e934aa6 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 6 Mar 2019 12:08:09 +0530 Subject: [PATCH 1/4] docs for scrapy.logformatter --- docs/topics/logging.rst | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 0986929ad..a5fecebba 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,6 +193,45 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +Custom Log Formats +------------------- + +Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class. + +Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from +``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing +the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message. +Listed below is details of what each key represents : + +* ``level`` is the log level for that action, you can use those from the python logging library: + :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR` + and :setting:`logging.CRITICAL`. + +* ``msg`` should be a string that can contain different formatting placeholders. This string, formatted + with the provided ``args``, is going to be the long message for that action. + +* ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is + computed as ``msg % args``. + +.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line + ``LOG_FORMATTER = '’`` + +.. class:: scrapy.logformatter.LogFormatter + + The default log formatting class in Scrapy. + + .. method:: crawled (request, response, spider) + + ``crawled`` is called to log message when the crawler finds a webpage. + + .. method:: scraped(item, response, spider) + + ``scraped`` is called to log message when an item scraped by a spider. + + .. method:: dropped(item, exception, response, spider) + + ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. + Advanced customization ---------------------- From 924b67437b92f14601816d02c5d153e7281da6d4 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 7 Mar 2019 16:40:59 +0530 Subject: [PATCH 2/4] move api docs to source code --- docs/topics/logging.rst | 38 ++++---------------------------------- docs/topics/settings.rst | 9 +++++++++ scrapy/logformatter.py | 36 +++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 47 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a5fecebba..72f24bae6 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -196,41 +196,11 @@ to override some of the Scrapy settings regarding logging. Custom Log Formats ------------------- -Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class. - -Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from -``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing -the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message. -Listed below is details of what each key represents : - -* ``level`` is the log level for that action, you can use those from the python logging library: - :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR` - and :setting:`logging.CRITICAL`. - -* ``msg`` should be a string that can contain different formatting placeholders. This string, formatted - with the provided ``args``, is going to be the long message for that action. - -* ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is - computed as ``msg % args``. - -.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line - ``LOG_FORMATTER = '’`` +Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class. -.. class:: scrapy.logformatter.LogFormatter - - The default log formatting class in Scrapy. - - .. method:: crawled (request, response, spider) - - ``crawled`` is called to log message when the crawler finds a webpage. - - .. method:: scraped(item, response, spider) - - ``scraped`` is called to log message when an item scraped by a spider. - - .. method:: dropped(item, exception, response, spider) - - ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. +.. autoclass:: scrapy.logformatter.LogFormatter + :members: Advanced customization ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..1dfb5b8aa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -866,6 +866,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: ``scrapy.logformatter.LogFormatter`` + +The class to use for formatting log messages for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 075a6d862..0bb8aee58 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,25 +13,29 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - All methods must return a dictionary listing the parameters `level`, `msg` - and `args` which are going to be used for constructing the log message when - calling logging.log. + All methods must return a dictionary listing the parameters ``level``, ``msg`` + and ``args`` which are going to be used for constructing the log message when + calling ``logging.log``. Dictionary keys for the method outputs: - * `level` should be the log level for that action, you can use those - from the python logging library: logging.DEBUG, logging.INFO, - logging.WARNING, logging.ERROR and logging.CRITICAL. - * `msg` should be a string that can contain different formatting - placeholders. This string, formatted with the provided `args`, is going - to be the log message for that action. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + + * ``msg`` should be a string that can contain different formatting placeholders. This string, formatted + with the provided ``args``, is going to be the long message for that action. + + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is + computed as ``msg % args``. - * `args` should be a tuple or dict with the formatting placeholders for - `msg`. The final log message is computed as output['msg'] % - output['args']. """ def crawled(self, request, response, spider): + """ + ``crawled`` is called to log message when the crawler finds a webpage. + """ request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -40,7 +44,7 @@ class LogFormatter(object): 'args': { 'status': response.status, 'request': request, - 'request_flags' : request_flags, + 'request_flags': request_flags, 'referer': referer_str(request), 'response_flags': response_flags, # backward compatibility with Scrapy logformatter below 1.4 version @@ -49,6 +53,9 @@ class LogFormatter(object): } def scraped(self, item, response, spider): + """ + ``scraped`` is called to log message when an item is scraped by a spider. + """ if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -63,6 +70,9 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): + """ + ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. + """ return { 'level': logging.WARNING, 'msg': DROPPEDMSG, From 82049e9c41f878d84f0fe10f827c6fe2a33f7ba6 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sun, 10 Mar 2019 20:14:55 +0530 Subject: [PATCH 3/4] make suggested changes. --- docs/topics/logging.rst | 10 ++++++---- docs/topics/settings.rst | 4 ++-- scrapy/logformatter.py | 26 +++++++++++++++++--------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 72f24bae6..006530a8c 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,11 +193,13 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers -Custom Log Formats -------------------- +.. _custom-log-formats: -Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class -and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class. +Custom Log Formats +------------------ + +A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` point to your new class. .. autoclass:: scrapy.logformatter.LogFormatter :members: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1dfb5b8aa..a36c0b34c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -871,9 +871,9 @@ directives. LOG_FORMATTER ------------- -Default: ``scrapy.logformatter.LogFormatter`` +Default: :class:`scrapy.logformatter.LogFormatter` -The class to use for formatting log messages for different actions. +The class to use for :ref:`formatting log messages ` for different actions. .. setting:: LOG_LEVEL diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 0bb8aee58..17c69cba8 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -30,12 +30,24 @@ class LogFormatter(object): * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is computed as ``msg % args``. + Here is an example on how to create a custom log formatter to lower the severity level of the log message + when an item is dropped from the pipeline:: + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + 'level': logging.INFO, # lowering the level from logging.WARNING + 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'args': { + 'exception': exception, + 'item': item, + } + } + """ def crawled(self, request, response, spider): - """ - ``crawled`` is called to log message when the crawler finds a webpage. - """ + """Logs a message when the crawler finds a webpage.""" request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -53,9 +65,7 @@ class LogFormatter(object): } def scraped(self, item, response, spider): - """ - ``scraped`` is called to log message when an item is scraped by a spider. - """ + """Logs a message when an item is scraped by a spider.""" if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -70,9 +80,7 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): - """ - ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. - """ + """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { 'level': logging.WARNING, 'msg': DROPPEDMSG, From e9cd4ee03aa41e27bea0408b10970ec5bedf35d3 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sun, 10 Mar 2019 20:37:56 +0530 Subject: [PATCH 4/4] fix list alignment and line width --- scrapy/logformatter.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 17c69cba8..717120242 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -19,19 +19,18 @@ class LogFormatter(object): Dictionary keys for the method outputs: - * ``level`` is the log level for that action, you can use those from the - `python logging library `_ : - ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` - and ``logging.CRITICAL``. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + * ``msg`` should be a string that can contain different formatting placeholders. + This string, formatted with the provided ``args``, is going to be the long message + for that action. + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. + The final log message is computed as ``msg % args``. - * ``msg`` should be a string that can contain different formatting placeholders. This string, formatted - with the provided ``args``, is going to be the long message for that action. - - * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is - computed as ``msg % args``. - - Here is an example on how to create a custom log formatter to lower the severity level of the log message - when an item is dropped from the pipeline:: + Here is an example on how to create a custom log formatter to lower the severity level of + the log message when an item is dropped from the pipeline:: class PoliteLogFormatter(logformatter.LogFormatter): def dropped(self, item, exception, response, spider):