diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index d79c1eda2..a4d92faa9 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -101,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' ITEM_PIPELINES = [] LOG_ENABLED = True +LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None diff --git a/scrapy/contrib/logformatter.py b/scrapy/contrib/logformatter.py new file mode 100644 index 000000000..55995f6e8 --- /dev/null +++ b/scrapy/contrib/logformatter.py @@ -0,0 +1,7 @@ +"""Functions for logging diferent actions""" + +def crawled_logline(request, response): + referer = request.headers.get('Referer') + flags = ' %s' % str(response.flags) if response.flags else '' + return "Crawled (%d) %s (referer: %s)%s" % (response.status, \ + request, referer, flags) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f06e4a374..aaaed7fd9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,6 +34,7 @@ class ExecutionEngine(object): self.paused = False self._next_request_pending = set() self._mainloop_task = task.LoopingCall(self._mainloop) + self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED']) def configure(self): """ @@ -198,15 +199,14 @@ class ExecutionEngine(object): def download(self, request, spider): domain = spider.domain_name - referer = request.headers.get('Referer') def _on_success(response): """handle the result of a page download""" assert isinstance(response, (Response, Request)) if isinstance(response, Response): response.request = request # tie request to response received - log.msg("Crawled %s (referer: <%s>)" % (request, referer), \ - level=log.DEBUG, domain=domain) + log.msg(self._crawled_logline(request, response), \ + level=log.DEBUG, domain=spider.domain_name) return response elif isinstance(response, Request): newrequest = response @@ -224,8 +224,8 @@ class ExecutionEngine(object): errmsg = str(_failure) level = log.ERROR if errmsg: - log.msg("Downloading <%s> (referer: <%s>): %s" % (request.url, \ - referer, errmsg), level=level, domain=domain) + log.msg("Crawling <%s>: %s" % (request.url, errmsg), \ + level=level, domain=domain) return Failure(IgnoreRequest(str(exc))) def _on_complete(_): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 8cd55fe99..65355b03d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -85,10 +85,7 @@ class Request(object_ref): return self._encoding def __str__(self): - if self.method == 'GET': - return "<%s>" % self.url - else: - return "<%s %s>" % (self.method, self.url) + return "<%s %s>" % (self.method, self.url) def __repr__(self): attrs = ['url', 'method', 'body', 'headers', 'cookies', 'meta'] diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 11a9bfcde..f632ea117 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -65,9 +65,7 @@ class Response(object_ref): return "%s(%s)" % (self.__class__.__name__, args) def __str__(self): - flags = "(%s) " % ",".join(self.flags) if self.flags else "" - status = "%d " % self.status + " " if self.status != 200 else "" - return "%s<%s%s>" % (flags, status, self.url) + return "<%d %s>" % (self.status, self.url) def copy(self): """Return a copy of this Response""" diff --git a/scrapy/tests/test_contrib_logformatter.py b/scrapy/tests/test_contrib_logformatter.py new file mode 100644 index 000000000..e096f7359 --- /dev/null +++ b/scrapy/tests/test_contrib_logformatter.py @@ -0,0 +1,22 @@ +import unittest + +from scrapy.http import Request, Response +from scrapy.contrib.logformatter import crawled_logline + + +class LoggingContribTest(unittest.TestCase): + + def test_crawled_logline(self): + req = Request("http://www.example.com") + res = Response("http://www.example.com") + self.assertEqual(crawled_logline(req, res), + "Crawled (200) (referer: None)") + + req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) + res = Response("http://www.example.com", flags=['cached']) + self.assertEqual(crawled_logline(req, res), + "Crawled (200) (referer: http://example.com) ['cached']") + + +if __name__ == "__main__": + unittest.main()