mirror of https://github.com/scrapy/scrapy.git
Deprecate scrapy/log.py
This commit is contained in:
Julia Medina
parent
6acb3848fb
commit
c174d78f12
165
scrapy/log.py
165
scrapy/log.py
|
|
@ -1,161 +1,10 @@
|
|||
"""
|
||||
Scrapy logging facility
|
||||
This module is kept to provide a helpful warning about its removal.
|
||||
"""
|
||||
|
||||
See documentation in docs/topics/logging.rst
|
||||
"""
|
||||
import sys
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
from twisted.python import log
|
||||
|
||||
import scrapy
|
||||
from scrapy.utils.python import unicode_to_str
|
||||
from scrapy.settings import overridden_settings
|
||||
|
||||
# Logging levels
|
||||
DEBUG = logging.DEBUG
|
||||
INFO = logging.INFO
|
||||
WARNING = logging.WARNING
|
||||
ERROR = logging.ERROR
|
||||
CRITICAL = logging.CRITICAL
|
||||
SILENT = CRITICAL + 1
|
||||
|
||||
level_names = {
|
||||
logging.DEBUG: "DEBUG",
|
||||
logging.INFO: "INFO",
|
||||
logging.WARNING: "WARNING",
|
||||
logging.ERROR: "ERROR",
|
||||
logging.CRITICAL: "CRITICAL",
|
||||
SILENT: "SILENT",
|
||||
}
|
||||
|
||||
class ScrapyFileLogObserver(log.FileLogObserver):
|
||||
|
||||
def __init__(self, f, level=INFO, encoding='utf-8', crawler=None):
|
||||
self.level = level
|
||||
self.encoding = encoding
|
||||
self.crawler = crawler
|
||||
if crawler:
|
||||
self.emit = self._emit_with_crawler
|
||||
else:
|
||||
self.emit = self._emit
|
||||
log.FileLogObserver.__init__(self, f)
|
||||
|
||||
def _emit(self, eventDict):
|
||||
ev = _adapt_eventdict(eventDict, self.level, self.encoding,
|
||||
self.crawler)
|
||||
if ev is not None:
|
||||
log.FileLogObserver.emit(self, ev)
|
||||
return ev
|
||||
|
||||
def _emit_with_crawler(self, eventDict):
|
||||
ev = self._emit(eventDict)
|
||||
if ev:
|
||||
level = ev['logLevel']
|
||||
sname = 'log_count/%s' % level_names.get(level, level)
|
||||
self.crawler.stats.inc_value(sname)
|
||||
|
||||
def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8',
|
||||
crawler=None, prepend_level=True):
|
||||
"""Adapt Twisted log eventDict making it suitable for logging with a Scrapy
|
||||
log observer. It may return None to indicate that the event should be
|
||||
ignored by a Scrapy log observer.
|
||||
|
||||
`log_level` is the minimum level being logged, and `encoding` is the log
|
||||
encoding.
|
||||
"""
|
||||
ev = eventDict.copy()
|
||||
if ev['isError']:
|
||||
ev.setdefault('logLevel', ERROR)
|
||||
|
||||
# ignore non-error messages from outside scrapy
|
||||
if ev.get('system') != 'scrapy' and not ev['isError']:
|
||||
return
|
||||
|
||||
level = ev.get('logLevel')
|
||||
if level < log_level:
|
||||
return
|
||||
|
||||
spider = ev.get('spider')
|
||||
if spider:
|
||||
ev['system'] = unicode_to_str(spider.name, encoding)
|
||||
if crawler and (not spider or spider.crawler is not crawler):
|
||||
# ignore events not triggered by own spiders in crawlers' observers
|
||||
return
|
||||
if not crawler and spider:
|
||||
# ignore spiders' events in observers without crawler
|
||||
return
|
||||
|
||||
lvlname = level_names.get(level, 'NOLEVEL')
|
||||
message = ev.get('message')
|
||||
if message:
|
||||
message = [unicode_to_str(x, encoding) for x in message]
|
||||
if prepend_level:
|
||||
message[0] = "%s: %s" % (lvlname, message[0])
|
||||
ev['message'] = message
|
||||
|
||||
why = ev.get('why')
|
||||
if why:
|
||||
why = unicode_to_str(why, encoding)
|
||||
if prepend_level:
|
||||
why = "%s: %s" % (lvlname, why)
|
||||
ev['why'] = why
|
||||
|
||||
fmt = ev.get('format')
|
||||
if fmt:
|
||||
fmt = unicode_to_str(fmt, encoding)
|
||||
if prepend_level:
|
||||
fmt = "%s: %s" % (lvlname, fmt)
|
||||
ev['format'] = fmt
|
||||
|
||||
return ev
|
||||
|
||||
def _get_log_level(level_name_or_id):
|
||||
if isinstance(level_name_or_id, int):
|
||||
return level_name_or_id
|
||||
elif isinstance(level_name_or_id, basestring):
|
||||
return globals()[level_name_or_id]
|
||||
else:
|
||||
raise ValueError("Unknown log level: %r" % level_name_or_id)
|
||||
|
||||
def start(logfile=None, loglevel='INFO', logstdout=True, logencoding='utf-8', crawler=None):
|
||||
loglevel = _get_log_level(loglevel)
|
||||
file = open(logfile, 'a') if logfile else sys.stderr
|
||||
log_observer = ScrapyFileLogObserver(file, loglevel, logencoding, crawler)
|
||||
_oldshowwarning = warnings.showwarning
|
||||
log.startLoggingWithObserver(log_observer.emit, setStdout=logstdout)
|
||||
# restore warnings, wrongly silenced by Twisted
|
||||
warnings.showwarning = _oldshowwarning
|
||||
return log_observer
|
||||
|
||||
def msg(message=None, _level=INFO, **kw):
|
||||
kw['logLevel'] = kw.pop('level', _level)
|
||||
kw.setdefault('system', 'scrapy')
|
||||
if message is None:
|
||||
log.msg(**kw)
|
||||
else:
|
||||
log.msg(message, **kw)
|
||||
|
||||
def err(_stuff=None, _why=None, **kw):
|
||||
kw['logLevel'] = kw.pop('level', ERROR)
|
||||
kw.setdefault('system', 'scrapy')
|
||||
log.err(_stuff, _why, **kw)
|
||||
|
||||
def start_from_settings(settings, crawler=None):
|
||||
if settings.getbool('LOG_ENABLED'):
|
||||
return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
|
||||
settings['LOG_ENCODING'], crawler)
|
||||
|
||||
def scrapy_info(settings):
|
||||
msg("Scrapy %s started (bot: %s)" % (scrapy.__version__,
|
||||
settings['BOT_NAME']))
|
||||
|
||||
msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
|
||||
level=INFO)
|
||||
|
||||
d = dict(overridden_settings(settings))
|
||||
msg(format="Overridden settings: %(settings)r", settings=d, level=INFO)
|
||||
|
||||
def start_from_crawler(crawler):
|
||||
return start_from_settings(crawler.settings, crawler)
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
|
||||
"the builtin Python library for logging. Read the updated "
|
||||
"logging entry in the documentation to learn more.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ tests/test_http_cookies.py
|
|||
tests/test_http_request.py
|
||||
tests/test_http_response.py
|
||||
tests/test_logformatter.py
|
||||
tests/test_log.py
|
||||
tests/test_mail.py
|
||||
tests/test_middleware.py
|
||||
tests/test_pipeline_files.py
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
from io import BytesIO
|
||||
|
||||
from twisted.python import log as txlog, failure
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.spider import Spider
|
||||
from scrapy.settings import default_settings
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
class LogTest(unittest.TestCase):
|
||||
|
||||
def test_get_log_level(self):
|
||||
default_log_level = getattr(log, default_settings.LOG_LEVEL)
|
||||
self.assertEqual(log._get_log_level('WARNING'), log.WARNING)
|
||||
self.assertEqual(log._get_log_level(log.WARNING), log.WARNING)
|
||||
self.assertRaises(ValueError, log._get_log_level, object())
|
||||
|
||||
class ScrapyFileLogObserverTest(unittest.TestCase):
|
||||
|
||||
level = log.INFO
|
||||
encoding = 'utf-8'
|
||||
|
||||
def setUp(self):
|
||||
self.f = BytesIO()
|
||||
self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding)
|
||||
self.log_observer.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.flushLoggedErrors()
|
||||
self.log_observer.stop()
|
||||
|
||||
def logged(self):
|
||||
return self.f.getvalue().strip()[25:]
|
||||
|
||||
def first_log_line(self):
|
||||
logged = self.logged()
|
||||
return logged.splitlines()[0] if logged else ''
|
||||
|
||||
def test_msg_basic(self):
|
||||
log.msg("Hello")
|
||||
self.assertEqual(self.logged(), "[scrapy] INFO: Hello")
|
||||
|
||||
def test_msg_ignore_spider(self):
|
||||
spider = Spider("myspider")
|
||||
log.msg("Hello", spider=spider)
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_msg_level1(self):
|
||||
log.msg("Hello", level=log.WARNING)
|
||||
self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
|
||||
|
||||
def test_msg_level2(self):
|
||||
log.msg("Hello", log.WARNING)
|
||||
self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
|
||||
|
||||
def test_msg_wrong_level(self):
|
||||
log.msg("Hello", level=9999)
|
||||
self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello")
|
||||
|
||||
def test_msg_encoding(self):
|
||||
log.msg(u"Price: \xa3100")
|
||||
self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100")
|
||||
|
||||
def test_msg_ignore_level(self):
|
||||
log.msg("Hello", level=log.DEBUG)
|
||||
log.msg("World", level=log.INFO)
|
||||
self.assertEqual(self.logged(), "[scrapy] INFO: World")
|
||||
|
||||
def test_msg_ignore_system(self):
|
||||
txlog.msg("Hello")
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_msg_ignore_system_err(self):
|
||||
txlog.msg("Hello")
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_err_noargs(self):
|
||||
try:
|
||||
a = 1/0
|
||||
except:
|
||||
log.err()
|
||||
self.assertIn('Traceback', self.logged())
|
||||
self.assertIn('ZeroDivisionError', self.logged())
|
||||
|
||||
def test_err_why(self):
|
||||
log.err(TypeError("bad type"), "Wrong type")
|
||||
self.assertEqual(self.first_log_line(), "[scrapy] ERROR: Wrong type")
|
||||
self.assertIn('TypeError', self.logged())
|
||||
self.assertIn('bad type', self.logged())
|
||||
|
||||
def test_error_outside_scrapy(self):
|
||||
"""Scrapy logger should still print outside errors"""
|
||||
txlog.err(TypeError("bad type"), "Wrong type")
|
||||
self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type")
|
||||
self.assertIn('TypeError', self.logged())
|
||||
self.assertIn('bad type', self.logged())
|
||||
|
||||
# this test fails in twisted trial observer, not in scrapy observer
|
||||
# def test_err_why_encoding(self):
|
||||
# log.err(TypeError("bad type"), u"\xa3")
|
||||
# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xc2\xa3")
|
||||
|
||||
def test_err_exc(self):
|
||||
log.err(TypeError("bad type"))
|
||||
self.assertIn('Unhandled Error', self.logged())
|
||||
self.assertIn('TypeError', self.logged())
|
||||
self.assertIn('bad type', self.logged())
|
||||
|
||||
def test_err_failure(self):
|
||||
log.err(failure.Failure(TypeError("bad type")))
|
||||
self.assertIn('Unhandled Error', self.logged())
|
||||
self.assertIn('TypeError', self.logged())
|
||||
self.assertIn('bad type', self.logged())
|
||||
|
||||
|
||||
class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest):
|
||||
|
||||
encoding = 'latin-1'
|
||||
|
||||
def test_msg_encoding(self):
|
||||
log.msg(u"Price: \xa3100")
|
||||
logged = self.f.getvalue().strip()[25:]
|
||||
self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xa3100")
|
||||
|
||||
# this test fails in twisted trial observer, not in scrapy observer
|
||||
# def test_err_why_encoding(self):
|
||||
# log.err(TypeError("bad type"), u"\xa3")
|
||||
# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3")
|
||||
|
||||
|
||||
class CrawlerScrapyFileLogObserverTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.f = BytesIO()
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler.spider = self.crawler._create_spider('test')
|
||||
self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO,
|
||||
'utf-8', self.crawler)
|
||||
self.log_observer.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.flushLoggedErrors()
|
||||
self.log_observer.stop()
|
||||
|
||||
def logged(self):
|
||||
return self.f.getvalue().strip()[25:]
|
||||
|
||||
def test_msg_basic(self):
|
||||
log.msg("Hello", spider=self.spider)
|
||||
self.assertEqual(self.logged(), "[test] INFO: Hello")
|
||||
|
||||
def test_msg_ignore_scrapy_channel(self):
|
||||
log.msg("Hello")
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_msg_ignore_another_crawler(self):
|
||||
crawler = get_crawler(Spider)
|
||||
log.msg("Hello", spider=crawler._create_spider('test'))
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_msg_stats_log(self):
|
||||
assert self.crawler.stats.get_value('log_count/INFO', 0) == 0
|
||||
log.msg("Hello", spider=self.spider)
|
||||
self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue