diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 0f01c441c..b1f5b8894 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -39,29 +39,16 @@ class PeriodicLog: interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") try: ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") - except ValueError: + except (TypeError, ValueError): ext_stats = ( {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_STATS") else None ) - except TypeError: - ext_stats = ( - {"enabled": True} - if crawler.settings.getbool("PERIODIC_LOG_STATS") - else None - ) - try: ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") - except ValueError: + except (TypeError, ValueError): ext_delta = ( - {"enabled": True} - if crawler.settings.getdict("PERIODIC_LOG_DELTA") - else None - ) - except TypeError: - ext_stats = ( {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_DELTA") else None diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 9f7ec7b23..4dccc687b 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,20 +51,78 @@ stats_dump_2 = { } +def extension(settings=None): + return PeriodicLog.from_crawler( + Crawler( + MetaSpider, + settings=settings, + ) + ) + + class TestPeriodicLog(unittest.TestCase): def test_extension_enabled(self): - extension = PeriodicLog.from_crawler( - Crawler( - MetaSpider, - settings={"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}, - ) - ) - # Test enabled - assert extension + # Expected that settings for this extension loaded succesfully + # And on certain conditions - extension raising NotConfigured + + # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} + # due to TypeError exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) + + # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} + # due to JSONDecodeError(ValueError) exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) + + # The ame for PERIODIC_LOG_DELTA: + assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) + assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) # Raise not configured if not set by settings with self.assertRaises(NotConfigured): - PeriodicLog.from_crawler(Crawler(MetaSpider)) + extension() + + # Regular use cases: + assert extension( + { + "PERIODIC_LOG_STATS": { + "include": [ + "downloader/", + "scheduler/", + "log_count/", + "item_scraped_count/", + ], + "exclude": ["scheduler/"], + } + } + ) + + assert extension( + { + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + "PERIODIC_LOG_TIMING_ENABLED": True, + } + ) + + assert extension( + { + "PERIODIC_LOG_TIMING_ENABLED": True, + } + ) + + assert extension( + { + "PERIODIC_LOG_STATS": { + "include": [ + "downloader/", + "scheduler/", + "log_count/", + "item_scraped_count/", + ], + "exclude": ["scheduler/"], + }, + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + } + ) def test_periodic_log_stats(self): pass