From 8baad552676de1ea0f3789a03279a025ed26f17e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 06:33:03 -0300
Subject: [PATCH] New scrapy/utils/log.py file with basic log helpers

There are two functions, `configure_logging` and `log_scrapy_info` which
intend to replace scrapy.log.start and scrapy.log.scrapy_info
respectively.

Creating new functions makes evident the backward incompatible change of
using another logging system, and since the Python logging module is a
standard builtin, additional helpers make sense to be on a scrapy/utils
file.
---
 scrapy/crawler.py   | 10 +++-------
 scrapy/utils/log.py | 48 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 7 deletions(-)
 create mode 100644 scrapy/utils/log.py

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f1ef1b524..f96086605 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -15,6 +15,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
+from scrapy.utils.log import configure_logging, log_scrapy_info
 from scrapy import signals
 
 logger = logging.getLogger('scrapy')
@@ -142,8 +143,8 @@ class CrawlerProcess(CrawlerRunner):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         self.stopping = False
-        self.log_observer = log.start_from_settings(self.settings)
-        log.scrapy_info(settings)
+        configure_logging(settings)
+        log_scrapy_info(settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -157,7 +158,6 @@ class CrawlerProcess(CrawlerRunner):
         signame = signal_names[signum]
         logger.info('Received %(signame)s twice, forcing unclean shutdown',
                     {'signame': signame})
-        self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True):
@@ -176,10 +176,6 @@ class CrawlerProcess(CrawlerRunner):
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _stop_logging(self):
-        if self.log_observer:
-            self.log_observer.stop()
-
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
new file mode 100644
index 000000000..3cdb543b9
--- /dev/null
+++ b/scrapy/utils/log.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import logging
+from logging.config import dictConfig
+
+from twisted.python import log as twisted_log
+
+import scrapy
+from scrapy.settings import overridden_settings
+
+logger = logging.getLogger('scrapy')
+
+
+DEFAULT_LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'loggers': {
+        'scrapy': {
+            'level': 'DEBUG',
+        },
+        'twisted': {
+            'level': 'ERROR',
+        },
+    }
+}
+
+
+def configure_logging(settings=None):
+    if not sys.warnoptions:
+        # Route warnings through python logging
+        logging.captureWarnings(True)
+
+    observer = twisted_log.PythonLoggingObserver('twisted')
+    observer.start()
+
+    dictConfig(DEFAULT_LOGGING)
+
+
+def log_scrapy_info(settings):
+    logger.info("Scrapy %(version)s started (bot: %(bot)s)",
+                {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
+
+    logger.info("Optional features available: %(features)s",
+                {'features': ", ".join(scrapy.optional_features)})
+
+    d = dict(overridden_settings(settings))
+    logger.info("Overridden settings: %(settings)r", {'settings': d})