From c174d78f1221d65affcd94b72cfccee8d63cdf1b Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Sat, 28 Feb 2015 09:30:28 -0300 Subject: [PATCH] Deprecate scrapy/log.py --- scrapy/log.py | 165 ++--------------------------------------- tests/py3-ignores.txt | 1 - tests/test_log.py | 169 ------------------------------------------ 3 files changed, 7 insertions(+), 328 deletions(-) delete mode 100644 tests/test_log.py diff --git a/scrapy/log.py b/scrapy/log.py index aa53e3574..f87bf6461 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -1,161 +1,10 @@ """ -Scrapy logging facility +This module is kept to provide a helpful warning about its removal. +""" -See documentation in docs/topics/logging.rst -""" -import sys -import logging import warnings - -from twisted.python import log - -import scrapy -from scrapy.utils.python import unicode_to_str -from scrapy.settings import overridden_settings - -# Logging levels -DEBUG = logging.DEBUG -INFO = logging.INFO -WARNING = logging.WARNING -ERROR = logging.ERROR -CRITICAL = logging.CRITICAL -SILENT = CRITICAL + 1 - -level_names = { - logging.DEBUG: "DEBUG", - logging.INFO: "INFO", - logging.WARNING: "WARNING", - logging.ERROR: "ERROR", - logging.CRITICAL: "CRITICAL", - SILENT: "SILENT", -} - -class ScrapyFileLogObserver(log.FileLogObserver): - - def __init__(self, f, level=INFO, encoding='utf-8', crawler=None): - self.level = level - self.encoding = encoding - self.crawler = crawler - if crawler: - self.emit = self._emit_with_crawler - else: - self.emit = self._emit - log.FileLogObserver.__init__(self, f) - - def _emit(self, eventDict): - ev = _adapt_eventdict(eventDict, self.level, self.encoding, - self.crawler) - if ev is not None: - log.FileLogObserver.emit(self, ev) - return ev - - def _emit_with_crawler(self, eventDict): - ev = self._emit(eventDict) - if ev: - level = ev['logLevel'] - sname = 'log_count/%s' % level_names.get(level, level) - self.crawler.stats.inc_value(sname) - -def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', - crawler=None, prepend_level=True): - """Adapt Twisted log eventDict making it suitable for logging with a Scrapy - log observer. It may return None to indicate that the event should be - ignored by a Scrapy log observer. - - `log_level` is the minimum level being logged, and `encoding` is the log - encoding. - """ - ev = eventDict.copy() - if ev['isError']: - ev.setdefault('logLevel', ERROR) - - # ignore non-error messages from outside scrapy - if ev.get('system') != 'scrapy' and not ev['isError']: - return - - level = ev.get('logLevel') - if level < log_level: - return - - spider = ev.get('spider') - if spider: - ev['system'] = unicode_to_str(spider.name, encoding) - if crawler and (not spider or spider.crawler is not crawler): - # ignore events not triggered by own spiders in crawlers' observers - return - if not crawler and spider: - # ignore spiders' events in observers without crawler - return - - lvlname = level_names.get(level, 'NOLEVEL') - message = ev.get('message') - if message: - message = [unicode_to_str(x, encoding) for x in message] - if prepend_level: - message[0] = "%s: %s" % (lvlname, message[0]) - ev['message'] = message - - why = ev.get('why') - if why: - why = unicode_to_str(why, encoding) - if prepend_level: - why = "%s: %s" % (lvlname, why) - ev['why'] = why - - fmt = ev.get('format') - if fmt: - fmt = unicode_to_str(fmt, encoding) - if prepend_level: - fmt = "%s: %s" % (lvlname, fmt) - ev['format'] = fmt - - return ev - -def _get_log_level(level_name_or_id): - if isinstance(level_name_or_id, int): - return level_name_or_id - elif isinstance(level_name_or_id, basestring): - return globals()[level_name_or_id] - else: - raise ValueError("Unknown log level: %r" % level_name_or_id) - -def start(logfile=None, loglevel='INFO', logstdout=True, logencoding='utf-8', crawler=None): - loglevel = _get_log_level(loglevel) - file = open(logfile, 'a') if logfile else sys.stderr - log_observer = ScrapyFileLogObserver(file, loglevel, logencoding, crawler) - _oldshowwarning = warnings.showwarning - log.startLoggingWithObserver(log_observer.emit, setStdout=logstdout) - # restore warnings, wrongly silenced by Twisted - warnings.showwarning = _oldshowwarning - return log_observer - -def msg(message=None, _level=INFO, **kw): - kw['logLevel'] = kw.pop('level', _level) - kw.setdefault('system', 'scrapy') - if message is None: - log.msg(**kw) - else: - log.msg(message, **kw) - -def err(_stuff=None, _why=None, **kw): - kw['logLevel'] = kw.pop('level', ERROR) - kw.setdefault('system', 'scrapy') - log.err(_stuff, _why, **kw) - -def start_from_settings(settings, crawler=None): - if settings.getbool('LOG_ENABLED'): - return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'], - settings['LOG_ENCODING'], crawler) - -def scrapy_info(settings): - msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, - settings['BOT_NAME'])) - - msg("Optional features available: %s" % ", ".join(scrapy.optional_features), - level=INFO) - - d = dict(overridden_settings(settings)) - msg(format="Overridden settings: %(settings)r", settings=d, level=INFO) - -def start_from_crawler(crawler): - return start_from_settings(crawler.settings, crawler) +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on " + "the builtin Python library for logging. Read the updated " + "logging entry in the documentation to learn more.", + ScrapyDeprecationWarning, stacklevel=2) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 0fc90eddb..d27d5dd3c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -31,7 +31,6 @@ tests/test_http_cookies.py tests/test_http_request.py tests/test_http_response.py tests/test_logformatter.py -tests/test_log.py tests/test_mail.py tests/test_middleware.py tests/test_pipeline_files.py diff --git a/tests/test_log.py b/tests/test_log.py deleted file mode 100644 index 113d10004..000000000 --- a/tests/test_log.py +++ /dev/null @@ -1,169 +0,0 @@ -from io import BytesIO - -from twisted.python import log as txlog, failure -from twisted.trial import unittest - -from scrapy import log -from scrapy.spider import Spider -from scrapy.settings import default_settings -from scrapy.utils.test import get_crawler - -class LogTest(unittest.TestCase): - - def test_get_log_level(self): - default_log_level = getattr(log, default_settings.LOG_LEVEL) - self.assertEqual(log._get_log_level('WARNING'), log.WARNING) - self.assertEqual(log._get_log_level(log.WARNING), log.WARNING) - self.assertRaises(ValueError, log._get_log_level, object()) - -class ScrapyFileLogObserverTest(unittest.TestCase): - - level = log.INFO - encoding = 'utf-8' - - def setUp(self): - self.f = BytesIO() - self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding) - self.log_observer.start() - - def tearDown(self): - self.flushLoggedErrors() - self.log_observer.stop() - - def logged(self): - return self.f.getvalue().strip()[25:] - - def first_log_line(self): - logged = self.logged() - return logged.splitlines()[0] if logged else '' - - def test_msg_basic(self): - log.msg("Hello") - self.assertEqual(self.logged(), "[scrapy] INFO: Hello") - - def test_msg_ignore_spider(self): - spider = Spider("myspider") - log.msg("Hello", spider=spider) - self.failIf(self.logged()) - - def test_msg_level1(self): - log.msg("Hello", level=log.WARNING) - self.assertEqual(self.logged(), "[scrapy] WARNING: Hello") - - def test_msg_level2(self): - log.msg("Hello", log.WARNING) - self.assertEqual(self.logged(), "[scrapy] WARNING: Hello") - - def test_msg_wrong_level(self): - log.msg("Hello", level=9999) - self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello") - - def test_msg_encoding(self): - log.msg(u"Price: \xa3100") - self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100") - - def test_msg_ignore_level(self): - log.msg("Hello", level=log.DEBUG) - log.msg("World", level=log.INFO) - self.assertEqual(self.logged(), "[scrapy] INFO: World") - - def test_msg_ignore_system(self): - txlog.msg("Hello") - self.failIf(self.logged()) - - def test_msg_ignore_system_err(self): - txlog.msg("Hello") - self.failIf(self.logged()) - - def test_err_noargs(self): - try: - a = 1/0 - except: - log.err() - self.assertIn('Traceback', self.logged()) - self.assertIn('ZeroDivisionError', self.logged()) - - def test_err_why(self): - log.err(TypeError("bad type"), "Wrong type") - self.assertEqual(self.first_log_line(), "[scrapy] ERROR: Wrong type") - self.assertIn('TypeError', self.logged()) - self.assertIn('bad type', self.logged()) - - def test_error_outside_scrapy(self): - """Scrapy logger should still print outside errors""" - txlog.err(TypeError("bad type"), "Wrong type") - self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type") - self.assertIn('TypeError', self.logged()) - self.assertIn('bad type', self.logged()) - -# this test fails in twisted trial observer, not in scrapy observer -# def test_err_why_encoding(self): -# log.err(TypeError("bad type"), u"\xa3") -# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xc2\xa3") - - def test_err_exc(self): - log.err(TypeError("bad type")) - self.assertIn('Unhandled Error', self.logged()) - self.assertIn('TypeError', self.logged()) - self.assertIn('bad type', self.logged()) - - def test_err_failure(self): - log.err(failure.Failure(TypeError("bad type"))) - self.assertIn('Unhandled Error', self.logged()) - self.assertIn('TypeError', self.logged()) - self.assertIn('bad type', self.logged()) - - -class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest): - - encoding = 'latin-1' - - def test_msg_encoding(self): - log.msg(u"Price: \xa3100") - logged = self.f.getvalue().strip()[25:] - self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xa3100") - -# this test fails in twisted trial observer, not in scrapy observer -# def test_err_why_encoding(self): -# log.err(TypeError("bad type"), u"\xa3") -# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3") - - -class CrawlerScrapyFileLogObserverTest(unittest.TestCase): - - def setUp(self): - self.f = BytesIO() - self.crawler = get_crawler(Spider) - self.spider = self.crawler.spider = self.crawler._create_spider('test') - self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO, - 'utf-8', self.crawler) - self.log_observer.start() - - def tearDown(self): - self.flushLoggedErrors() - self.log_observer.stop() - - def logged(self): - return self.f.getvalue().strip()[25:] - - def test_msg_basic(self): - log.msg("Hello", spider=self.spider) - self.assertEqual(self.logged(), "[test] INFO: Hello") - - def test_msg_ignore_scrapy_channel(self): - log.msg("Hello") - self.failIf(self.logged()) - - def test_msg_ignore_another_crawler(self): - crawler = get_crawler(Spider) - log.msg("Hello", spider=crawler._create_spider('test')) - self.failIf(self.logged()) - - def test_msg_stats_log(self): - assert self.crawler.stats.get_value('log_count/INFO', 0) == 0 - log.msg("Hello", spider=self.spider) - self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1) - - -if __name__ == "__main__": - unittest.main()