New scrapy/utils/log.py file with basic log helpers

There are two functions, `configure_logging` and `log_scrapy_info` which
intend to replace scrapy.log.start and scrapy.log.scrapy_info
respectively.

Creating new functions makes evident the backward incompatible change of
using another logging system, and since the Python logging module is a
standard builtin, additional helpers make sense to be on a scrapy/utils
file.

scrapy/crawler.py

@@ -15,6 +15,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
+from scrapy.utils.log import configure_logging, log_scrapy_info
 from scrapy import signals
 
 logger = logging.getLogger('scrapy')
@@ -142,8 +143,8 @@ class CrawlerProcess(CrawlerRunner):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         self.stopping = False
-        self.log_observer = log.start_from_settings(self.settings)
-        log.scrapy_info(settings)
+        configure_logging(settings)
+        log_scrapy_info(settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -157,7 +158,6 @@ class CrawlerProcess(CrawlerRunner):
         signame = signal_names[signum]
         logger.info('Received %(signame)s twice, forcing unclean shutdown',
                     {'signame': signame})
-        self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True):
@@ -176,10 +176,6 @@ class CrawlerProcess(CrawlerRunner):
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _stop_logging(self):
-        if self.log_observer:
-            self.log_observer.stop()
-
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()

scrapy/utils/log.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import logging
+from logging.config import dictConfig
+
+from twisted.python import log as twisted_log
+
+import scrapy
+from scrapy.settings import overridden_settings
+
+logger = logging.getLogger('scrapy')
+
+
+DEFAULT_LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'loggers': {
+        'scrapy': {
+            'level': 'DEBUG',
+        },
+        'twisted': {
+            'level': 'ERROR',
+        },
+    }
+}
+
+
+def configure_logging(settings=None):
+    if not sys.warnoptions:
+        # Route warnings through python logging
+        logging.captureWarnings(True)
+
+    observer = twisted_log.PythonLoggingObserver('twisted')
+    observer.start()
+
+    dictConfig(DEFAULT_LOGGING)
+
+
+def log_scrapy_info(settings):
+    logger.info("Scrapy %(version)s started (bot: %(bot)s)",
+                {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
+
+    logger.info("Optional features available: %(features)s",
+                {'features': ", ".join(scrapy.optional_features)})
+
+    d = dict(overridden_settings(settings))
+    logger.info("Overridden settings: %(settings)r", {'settings': d})