some improvements to 'Crawled' log lines, delegating the formatting to a pluggable function

scrapy/conf/default_settings.py

@@ -101,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 ITEM_PIPELINES = []
 
 LOG_ENABLED = True
+LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None

scrapy/contrib/logformatter.py

@@ -0,0 +1,7 @@
+"""Functions for logging diferent actions"""
+
+def crawled_logline(request, response):
+    referer = request.headers.get('Referer')
+    flags = ' %s' % str(response.flags) if response.flags else ''
+    return "Crawled (%d) %s (referer: %s)%s" % (response.status, \
+        request, referer, flags)

scrapy/core/engine.py

@@ -34,6 +34,7 @@ class ExecutionEngine(object):
         self.paused = False
         self._next_request_pending = set()
         self._mainloop_task = task.LoopingCall(self._mainloop)
+        self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED'])
 
     def configure(self):
         """
@@ -198,15 +199,14 @@ class ExecutionEngine(object):
 
     def download(self, request, spider):
         domain = spider.domain_name
-        referer = request.headers.get('Referer')
 
         def _on_success(response):
             """handle the result of a page download"""
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
                 response.request = request # tie request to response received
-                log.msg("Crawled %s (referer: <%s>)" % (request, referer), \
-                    level=log.DEBUG, domain=domain)
+                log.msg(self._crawled_logline(request, response), \
+                    level=log.DEBUG, domain=spider.domain_name)
                 return response
             elif isinstance(response, Request):
                 newrequest = response
@@ -224,8 +224,8 @@ class ExecutionEngine(object):
                 errmsg = str(_failure)
                 level = log.ERROR
             if errmsg:
-                log.msg("Downloading <%s> (referer: <%s>): %s" % (request.url, \
-                    referer, errmsg), level=level, domain=domain)
+                log.msg("Crawling <%s>: %s" % (request.url, errmsg), \
+                    level=level, domain=domain)
             return Failure(IgnoreRequest(str(exc)))
 
         def _on_complete(_):

scrapy/http/request/__init__.py

@@ -85,10 +85,7 @@ class Request(object_ref):
         return self._encoding
 
     def __str__(self):
-        if self.method == 'GET':
-            return "<%s>" % self.url
-        else:
-            return "<%s %s>" % (self.method, self.url)
+        return "<%s %s>" % (self.method, self.url)
 
     def __repr__(self):
         attrs = ['url', 'method', 'body', 'headers', 'cookies', 'meta']

scrapy/http/response/__init__.py

@@ -65,9 +65,7 @@ class Response(object_ref):
         return "%s(%s)" % (self.__class__.__name__, args)
 
     def __str__(self):
-        flags = "(%s) " % ",".join(self.flags) if self.flags else ""
-        status = "%d " % self.status + " " if self.status != 200 else ""
-        return "%s<%s%s>" % (flags, status, self.url)
+        return "<%d %s>" % (self.status, self.url)
 
     def copy(self):
         """Return a copy of this Response"""

scrapy/tests/test_contrib_logformatter.py

@@ -0,0 +1,22 @@
+import unittest
+
+from scrapy.http import Request, Response
+from scrapy.contrib.logformatter import crawled_logline
+
+
+class LoggingContribTest(unittest.TestCase):
+
+    def test_crawled_logline(self):
+        req = Request("http://www.example.com")
+        res = Response("http://www.example.com")
+        self.assertEqual(crawled_logline(req, res),
+            "Crawled (200) <GET http://www.example.com> (referer: None)")
+
+        req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
+        res = Response("http://www.example.com", flags=['cached'])
+        self.assertEqual(crawled_logline(req, res),
+            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+
+
+if __name__ == "__main__":
+    unittest.main()