From 80b96fc1951656fba19e4194389b3e7ad331956f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 8 Aug 2009 16:07:10 -0300 Subject: [PATCH] minor changes to referer logging when crawling --- scrapy/core/engine.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0f81e6a64..24cdf068a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -229,14 +229,14 @@ class ExecutionEngine(object): def download(self, request, spider): domain = spider.domain_name - referer = request.headers.get('Referer', None) + referer = request.headers.get('Referer') def _on_success(response): """handle the result of a page download""" assert isinstance(response, (Response, Request)) if isinstance(response, Response): response.request = request # tie request to response received - log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, \ + log.msg("Crawled %s (referer: <%s>)" % (response, referer), level=log.DEBUG, \ domain=domain) return response elif isinstance(response, Request): @@ -248,8 +248,10 @@ class ExecutionEngine(object): def _on_error(_failure): """handle an error processing a page""" ex = _failure.value - errmsg = str(_failure) if not isinstance(ex, IgnoreRequest) else _failure.getErrorMessage() - log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain) + errmsg = str(_failure) if not isinstance(ex, IgnoreRequest) \ + else _failure.getErrorMessage() + log.msg("Downloading <%s> (referer: <%s>): %s" % (request.url, referer, errmsg), \ + log.ERROR, domain=domain) return Failure(IgnoreRequest(str(ex))) def _on_complete(_):