LogFormatter: Add the ability to skip log messages (#3987)

scrapy/core/engine.py

@@ -233,10 +233,11 @@ class ExecutionEngine(object):
         def _on_success(response):
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
-                response.request = request # tie request to response received
+                response.request = request  # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signal=signals.response_received, \
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                self.signals.send_catch_log(signal=signals.response_received,
                     response=response, request=request, spider=spider)
             return response
 

scrapy/core/scraper.py

@@ -225,7 +225,8 @@ class Scraper(object):
             ex = output.value
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
@@ -238,7 +239,8 @@ class Scraper(object):
                     spider=spider, failure=output)
         else:
             logkws = self.logformatter.scraped(output, response, spider)
-            logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+            if logkws is not None:
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)

scrapy/logformatter.py

@@ -29,6 +29,10 @@ class LogFormatter(object):
     *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
         The final log message is computed as ``msg % args``.
 
+    Users can define their own ``LogFormatter`` class if they want to customise how
+    each action is logged or if they want to omit it entirely. In order to omit
+    logging an action the method must return ``None``.
+
     Here is an example on how to create a custom log formatter to lower the severity level of
     the log message when an item is dropped from the pipeline::
 

tests/test_logformatter.py

@@ -1,10 +1,18 @@
 import unittest
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TwistedTestCase
 import six
 
-from scrapy.spiders import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import DropItem
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
+from scrapy.spiders import Spider
+from tests.mockserver import MockServer
+from tests.spiders import ItemSpider
 
 
 class CustomItem(Item):
@@ -89,5 +97,61 @@ class LogformatterSubclassTest(LoggingContribTest):
         pass
 
 
+class SkipMessagesLogFormatter(LogFormatter):
+    def crawled(self, *args, **kwargs):
+        return None
+
+    def scraped(self, *args, **kwargs):
+        return None
+
+    def dropped(self, *args, **kwargs):
+        return None
+
+
+class DropSomeItemsPipeline(object):
+    drop = True
+
+    def process_item(self, item, spider):
+        if self.drop:
+            self.drop = False
+            raise DropItem("Ignoring item")
+        else:
+            self.drop = True
+
+class ShowOrSkipMessagesTestCase(TwistedTestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.base_settings = {
+            'LOG_LEVEL': 'DEBUG',
+            'ITEM_PIPELINES': {
+                __name__ + '.DropSomeItemsPipeline': 300,
+            },
+        }
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_show_messages(self):
+        crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertIn("Dropped: Ignoring item", str(lc))
+
+    @defer.inlineCallbacks
+    def test_skip_messages(self):
+        settings = self.base_settings.copy()
+        settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+        crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertNotIn("Dropped: Ignoring item", str(lc))
+
+
 if __name__ == "__main__":
     unittest.main()