diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f1ef1b524..f96086605 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -15,6 +15,7 @@ from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object +from scrapy.utils.log import configure_logging, log_scrapy_info from scrapy import signals logger = logging.getLogger('scrapy') @@ -142,8 +143,8 @@ class CrawlerProcess(CrawlerRunner): super(CrawlerProcess, self).__init__(settings) install_shutdown_handlers(self._signal_shutdown) self.stopping = False - self.log_observer = log.start_from_settings(self.settings) - log.scrapy_info(settings) + configure_logging(settings) + log_scrapy_info(settings) def _signal_shutdown(self, signum, _): install_shutdown_handlers(self._signal_kill) @@ -157,7 +158,6 @@ class CrawlerProcess(CrawlerRunner): signame = signal_names[signum] logger.info('Received %(signame)s twice, forcing unclean shutdown', {'signame': signame}) - self._stop_logging() reactor.callFromThread(self._stop_reactor) def start(self, stop_after_crawl=True): @@ -176,10 +176,6 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _stop_logging(self): - if self.log_observer: - self.log_observer.stop() - def _stop_reactor(self, _=None): try: reactor.stop() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py new file mode 100644 index 000000000..3cdb543b9 --- /dev/null +++ b/scrapy/utils/log.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +import sys +import logging +from logging.config import dictConfig + +from twisted.python import log as twisted_log + +import scrapy +from scrapy.settings import overridden_settings + +logger = logging.getLogger('scrapy') + + +DEFAULT_LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'loggers': { + 'scrapy': { + 'level': 'DEBUG', + }, + 'twisted': { + 'level': 'ERROR', + }, + } +} + + +def configure_logging(settings=None): + if not sys.warnoptions: + # Route warnings through python logging + logging.captureWarnings(True) + + observer = twisted_log.PythonLoggingObserver('twisted') + observer.start() + + dictConfig(DEFAULT_LOGGING) + + +def log_scrapy_info(settings): + logger.info("Scrapy %(version)s started (bot: %(bot)s)", + {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) + + logger.info("Optional features available: %(features)s", + {'features': ", ".join(scrapy.optional_features)}) + + d = dict(overridden_settings(settings)) + logger.info("Overridden settings: %(settings)r", {'settings': d})