Merge pull request #5279 from erikkemperman/option-logfile-truncate

Add LOG_FILE_APPEND to settings

docs/topics/logging.rst

@@ -143,6 +143,7 @@ Logging settings
 These settings can be used to configure the logging:
 
 * :setting:`LOG_FILE`
+* :setting:`LOG_FILE_APPEND`
 * :setting:`LOG_ENABLED`
 * :setting:`LOG_ENCODING`
 * :setting:`LOG_LEVEL`
@@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
 :setting:`LOG_FILE` is set, messages sent through the root logger will be
 redirected to a file named :setting:`LOG_FILE` with encoding
 :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
-messages will be displayed on the standard error. Lastly, if
+messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
+and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
+(discarding the output from previous runs, if any). Lastly, if
 :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
 
 :setting:`LOG_LEVEL` determines the minimum level of severity to display, those

docs/topics/settings.rst

@@ -1026,6 +1026,16 @@ Default: ``None``
 
 File name to use for logging output. If ``None``, standard error will be used.
 
+.. setting:: LOG_FILE_APPEND
+
+LOG_FILE_APPEND
+---------------
+
+Default: ``True``
+
+If ``False``, the log file specified with :setting:`LOG_FILE` will be
+overwritten (discarding the output from previous runs, if any).
+
 .. setting:: LOG_FORMAT
 
 LOG_FORMAT

scrapy/settings/default_settings.py

@@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
+LOG_FILE_APPEND = True
 LOG_SHORT_NAMES = False
 
 SCHEDULER_DEBUG = False

scrapy/utils/log.py

@@ -124,8 +124,9 @@ def _get_handler(settings):
     """ Return a log handler object according to settings """
     filename = settings.get('LOG_FILE')
     if filename:
+        mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
         encoding = settings.get('LOG_ENCODING')
-        handler = logging.FileHandler(filename, encoding=encoding)
+        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
     elif settings.getbool('LOG_ENABLED'):
         handler = logging.StreamHandler()
     else:

tests/test_crawler.py

@@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
     def test_spider_custom_settings_log_level(self):
         log_file = self.mktemp()
+        with open(log_file, 'wb') as fo:
+            fo.write('previous message\n'.encode('utf-8'))
 
         class MySpider(scrapy.Spider):
             name = 'spider'
@@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         logging.error('error message')
 
         with open(log_file, 'rb') as fo:
-            logged = fo.read().decode('utf8')
+            logged = fo.read().decode('utf-8')
 
+        self.assertIn('previous message', logged)
         self.assertNotIn('debug message', logged)
         self.assertIn('info message', logged)
         self.assertIn('warning message', logged)
@@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             crawler.stats.get_value('log_count/INFO') - info_count, 1)
         self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
 
+    def test_spider_custom_settings_log_append(self):
+        log_file = self.mktemp()
+        with open(log_file, 'wb') as fo:
+            fo.write('previous message\n'.encode('utf-8'))
+
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'LOG_FILE': log_file,
+                'LOG_FILE_APPEND': False,
+                # disable telnet if not available to avoid an extra warning
+                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+            }
+
+        configure_logging()
+        Crawler(MySpider, {})
+        logging.debug('debug message')
+
+        with open(log_file, 'rb') as fo:
+            logged = fo.read().decode('utf-8')
+
+        self.assertNotIn('previous message', logged)
+        self.assertIn('debug message', logged)
+
 
 class SpiderLoaderWithWrongInterface:
 

tox.ini

@@ -57,6 +57,7 @@ deps =
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
     pytest-flake8
+    flake8==3.9.2  # https://github.com/tholo/pytest-flake8/issues/81
 commands =
     py.test --flake8 {posargs:docs scrapy tests}