From 736a4b615c08cc7e610ddb18f5ac782f0aedca61 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 10 Aug 2023 21:34:35 +0300 Subject: [PATCH 1/6] Update scrapy/settings/default_settings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9660e0bcd..f74b07c20 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -242,7 +242,7 @@ NEWSPIDER_MODULE = "" PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None -PERIODIC_LOG_TIMING_ENABLED = None +PERIODIC_LOG_TIMING_ENABLED = False RANDOMIZE_DOWNLOAD_DELAY = True From 2f094a7a5ca080ec6b43c1db4e0a70a385a0f756 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:14 +0300 Subject: [PATCH 2/6] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index ae94c55a4..e280224b1 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -481,6 +481,6 @@ Default: ``None`` PERIODIC_LOG_TIMING_ENABLED """"""""""""""""""""""""""" -Default: ``None`` +Default: ``False`` -``"PERIODIC_LOG_TIMING_ENABLED": True`` - enables logging of timing data \ No newline at end of file +``True`` enables logging of timing data (i.e. the ``"time"`` section). \ No newline at end of file From e6bd9829bddba9d3301d8c883e80632944a19e79 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:31 +0300 Subject: [PATCH 3/6] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index e280224b1..439e3e06e 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -471,9 +471,9 @@ PERIODIC_LOG_STATS Default: ``None`` -* ``"PERIODIC_LOG_STATS": True`` - show all available stats keys/values -* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - include stats for keys that have listed substrings in stats names. -* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. +* ``"PERIODIC_LOG_STATS": True`` - show the current value of all stats. +* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - show current values for stats with names containing any configured substring. +* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - show current values for all stats with names not containing any configured substring. .. setting:: PERIODIC_LOG_TIMING_ENABLED From 3a4a949f9d3d64eb7124545813cf46cf63910149 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:48 +0300 Subject: [PATCH 4/6] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 439e3e06e..e345618b6 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -460,9 +460,9 @@ PERIODIC_LOG_DELTA Default: ``None`` -* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stats values. -* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - include stats deltas for stats with names that have listed substrings in stats names. -* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. +* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stat values. +* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - show deltas for stats with names containing any configured substring. +* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - show deltas for all stats with names not containing any configured substring. .. setting:: PERIODIC_LOG_STATS From d67be20b2d1f9345b3b47bff8018e157c654142d Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:58 +0300 Subject: [PATCH 5/6] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index e345618b6..692ad72fe 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -426,15 +426,26 @@ Extension provides extended stats data periodically in addition to basic data fr } } -``"delta"`` section shows numeric difference in stats values between current and previous log entry with period of ``LOGSTATS_INTERVAL`` (60 seconds by default). Its applicable for stats with values types ``int`` and ``float``. -Stats values displayed in this section configured by :setting:`PERIODIC_LOG_DELTA` setting. +This extension logs the following configurable sections: -``"stats"`` section shows stats values as is at the moment of current period. -Stats values displayed in this section configured by :setting:`PERIODIC_LOG_STATS` setting. +- ``"delta"`` shows how some numeric stats have changed since the last stats + log message. + + The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They + must have ``int`` or ``float`` values. -``"time"`` This extension produce log entries on startup, periodically, and on end of crawl. As final log entry produced earlier than ``LOGSTATS_INTERVAL`` value - detailed timing data required for more precise stats. +- ``"stats"`` shows the current value of some stats. -Configured by :setting:`PERIODIC_LOG_TIMING_ENABLED` + The :setting:`PERIODIC_LOG_STATS` setting determines the target stats. + +- ``"time"`` shows detailed timing data. + + The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or + not to show this section. + +This extension logs data at the start, then on a fixed time interval +configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally +right before the crawl ends. Example extension configuration: From ac1694a9adc301c5bc2a340e034e0702b7d1a3fc Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:49:09 +0300 Subject: [PATCH 6/6] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 692ad72fe..3c1c7d16d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -389,7 +389,7 @@ Periodic log extension .. class:: PeriodicLog -Extension provides extended stats data periodically in addition to basic data from Log Stats and Core Stats extensions (as JSON compatible dictionary) like: :: +This extension periodically logs rich stat data as a JSON object:: 2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min) 2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: {