From 3d397de0e934233e70148c01afb78e63e2a03592 Mon Sep 17 00:00:00 2001 From: "Alexandr N Zamaraev (aka tonal)" Date: Fri, 28 Dec 2012 13:40:27 +0700 Subject: [PATCH 1/2] Possible set log-level in LogFormatter.dropped and LogFormatter.scraped --- scrapy/core/scraper.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 75090b258..332c47cca 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -201,14 +201,18 @@ class Scraper(object): ex = output.value if isinstance(ex, DropItem): logkws = self.logformatter.dropped(item, ex, response, spider) - log.msg(level=log.WARNING, spider=spider, **logkws) + if 'level' not in logkws: + logkws['level'] = log.WARNING + log.msg(spider=spider, **logkws) return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \ item=item, spider=spider, exception=output.value) else: log.err(output, 'Error processing %(item)s', item=item, spider=spider) else: logkws = self.logformatter.scraped(output, response, spider) - log.msg(level=log.DEBUG, spider=spider, **logkws) + if 'level' not in logkws: + logkws['level'] = log.DEBUG + log.msg(spider=spider, **logkws) return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \ item=output, response=response, spider=spider) From 71b071ffebc82dd0c6e1196f3d5f1e73389b8879 Mon Sep 17 00:00:00 2001 From: "Alexandr N Zamaraev (aka tonal)" Date: Sat, 29 Dec 2012 11:43:41 +0700 Subject: [PATCH 2/2] Log level return from LogFormatter methods --- scrapy/core/engine.py | 2 +- scrapy/core/scraper.py | 4 ---- scrapy/logformatter.py | 5 +++++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f87d7f5df..c544956a4 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -195,7 +195,7 @@ class ExecutionEngine(object): if isinstance(response, Response): response.request = request # tie request to response received logkws = self.logformatter.crawled(request, response, spider) - log.msg(level=log.DEBUG, spider=spider, **logkws) + log.msg(spider=spider, **logkws) self.signals.send_catch_log(signal=signals.response_received, \ response=response, request=request, spider=spider) return response diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 332c47cca..876e94176 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -201,8 +201,6 @@ class Scraper(object): ex = output.value if isinstance(ex, DropItem): logkws = self.logformatter.dropped(item, ex, response, spider) - if 'level' not in logkws: - logkws['level'] = log.WARNING log.msg(spider=spider, **logkws) return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \ item=item, spider=spider, exception=output.value) @@ -210,8 +208,6 @@ class Scraper(object): log.err(output, 'Error processing %(item)s', item=item, spider=spider) else: logkws = self.logformatter.scraped(output, response, spider) - if 'level' not in logkws: - logkws['level'] = log.DEBUG log.msg(spider=spider, **logkws) return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \ item=output, response=response, spider=spider) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 3a2afa103..d03d2d07f 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,6 +2,8 @@ import os from twisted.python.failure import Failure +from scrapy import log + SCRAPEDFMT = u"Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDFMT = u"Dropped: %(exception)s" + os.linesep + "%(item)s" @@ -16,6 +18,7 @@ class LogFormatter(object): def crawled(self, request, response, spider): flags = ' %s' % str(response.flags) if response.flags else '' return { + 'level': log.DEBUG, 'format': CRAWLEDFMT, 'status': response.status, 'request': request, @@ -26,6 +29,7 @@ class LogFormatter(object): def scraped(self, item, response, spider): src = response.getErrorMessage() if isinstance(response, Failure) else response return { + 'level': log.DEBUG, 'format': SCRAPEDFMT, 'src': src, 'item': item, @@ -33,6 +37,7 @@ class LogFormatter(object): def dropped(self, item, exception, response, spider): return { + 'level': log.WARNING, 'format': DROPPEDFMT, 'exception': exception, 'item': item,