From 0b52fa6ca9ec8916e4ebcaa7ee148e0b20c2068f Mon Sep 17 00:00:00 2001 From: watsta Date: Mon, 16 Sep 2019 14:12:04 +0200 Subject: [PATCH] LogFormatter: Add the ability to skip log messages (#3987) --- scrapy/core/engine.py | 7 ++-- scrapy/core/scraper.py | 6 ++-- scrapy/logformatter.py | 4 +++ tests/test_logformatter.py | 66 +++++++++++++++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 37fe0a873..fa913e528 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -233,10 +233,11 @@ class ExecutionEngine(object): def _on_success(response): assert isinstance(response, (Response, Request)) if isinstance(response, Response): - response.request = request # tie request to response received + response.request = request # tie request to response received logkws = self.logformatter.crawled(request, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signal=signals.response_received, \ + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + self.signals.send_catch_log(signal=signals.response_received, response=response, request=request, spider=spider) return response diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 66f5d0e05..1f389cf2e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -225,7 +225,8 @@ class Scraper(object): ex = output.value if isinstance(ex, DropItem): logkws = self.logformatter.dropped(item, ex, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) @@ -238,7 +239,8 @@ class Scraper(object): spider=spider, failure=output) else: logkws = self.logformatter.scraped(output, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) return self.signals.send_catch_log_deferred( signal=signals.item_scraped, item=output, response=response, spider=spider) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index b4d6787ff..f15940ed1 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -29,6 +29,10 @@ class LogFormatter(object): * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is computed as ``msg % args``. + Users can define their own ``LogFormatter`` class if they want to customise how + each action is logged or if they want to omit it entirely. In order to omit + logging an action the method must return ``None``. + Here is an example on how to create a custom log formatter to lower the severity level of the log message when an item is dropped from the pipeline:: diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 94e6c9fde..eb9c4a561 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,10 +1,18 @@ import unittest + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase as TwistedTestCase import six -from scrapy.spiders import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import DropItem from scrapy.http import Request, Response from scrapy.item import Item, Field from scrapy.logformatter import LogFormatter +from scrapy.spiders import Spider +from tests.mockserver import MockServer +from tests.spiders import ItemSpider class CustomItem(Item): @@ -89,5 +97,61 @@ class LogformatterSubclassTest(LoggingContribTest): pass +class SkipMessagesLogFormatter(LogFormatter): + def crawled(self, *args, **kwargs): + return None + + def scraped(self, *args, **kwargs): + return None + + def dropped(self, *args, **kwargs): + return None + + +class DropSomeItemsPipeline(object): + drop = True + + def process_item(self, item, spider): + if self.drop: + self.drop = False + raise DropItem("Ignoring item") + else: + self.drop = True + +class ShowOrSkipMessagesTestCase(TwistedTestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.base_settings = { + 'LOG_LEVEL': 'DEBUG', + 'ITEM_PIPELINES': { + __name__ + '.DropSomeItemsPipeline': 300, + }, + } + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_show_messages(self): + crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider) + with LogCapture() as lc: + yield crawler.crawl(mockserver=self.mockserver) + self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) + self.assertIn("Crawled (200)