RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Log level return from LogFormatter methods

This commit is contained in:
Alexandr N Zamaraev (aka tonal) 2012-12-29 11:43:41 +07:00
parent 3d397de0e9
commit 71b071ffeb
3 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/core/engine.py
View File

@ -195,7 +195,7 @@ class ExecutionEngine(object):
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
log.msg(level=log.DEBUG, spider=spider, **logkws)
log.msg(spider=spider, **logkws)
self.signals.send_catch_log(signal=signals.response_received, \
response=response, request=request, spider=spider)
return response

4
scrapy/core/scraper.py
View File

@ -201,8 +201,6 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
if 'level' not in logkws:
logkws['level'] = log.WARNING
log.msg(spider=spider, **logkws)
return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \
item=item, spider=spider, exception=output.value)
@ -210,8 +208,6 @@ class Scraper(object):
log.err(output, 'Error processing %(item)s', item=item, spider=spider)
else:
logkws = self.logformatter.scraped(output, response, spider)
if 'level' not in logkws:
logkws['level'] = log.DEBUG
log.msg(spider=spider, **logkws)
return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \
item=output, response=response, spider=spider)

5
scrapy/logformatter.py
View File

@ -2,6 +2,8 @@ import os
from twisted.python.failure import Failure
from scrapy import log
SCRAPEDFMT = u"Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDFMT = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
@ -16,6 +18,7 @@ class LogFormatter(object):
def crawled(self, request, response, spider):
flags = ' %s' % str(response.flags) if response.flags else ''
return {
'level': log.DEBUG,
'format': CRAWLEDFMT,
'status': response.status,
'request': request,
@ -26,6 +29,7 @@ class LogFormatter(object):
def scraped(self, item, response, spider):
src = response.getErrorMessage() if isinstance(response, Failure) else response
return {
'level': log.DEBUG,
'format': SCRAPEDFMT,
'src': src,
'item': item,
@ -33,6 +37,7 @@ class LogFormatter(object):
def dropped(self, item, exception, response, spider):
return {
'level': log.WARNING,
'format': DROPPEDFMT,
'exception': exception,
'item': item,