logformatter: support non-ascii characters in custom implementations of Item.__str__()

scrapy/logformatter.py

@@ -9,11 +9,11 @@ class LogFormatter(object):
     def crawled(self, request, response, spider):
         referer = request.headers.get('Referer')
         flags = ' %s' % str(response.flags) if response.flags else ''
-        return "Crawled (%d) %s (referer: %s)%s" % (response.status, \
+        return u"Crawled (%d) %s (referer: %s)%s" % (response.status, \
             request, referer, flags)
 
     def scraped(self, item, response, spider):
-        return "Scraped from %s" % response + os.linesep + str(item)
+        return u"Scraped from %s%s%s" % (response, os.linesep, item)
 
     def dropped(self, item, exception, response, spider):
-        return "Dropped: %s" % unicode(exception) + os.linesep + str(item)
+        return u"Dropped: %s%s%s" % (exception, os.linesep, item)

scrapy/tests/test_logformatter.py

@@ -2,9 +2,18 @@ import unittest
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
+from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
 
 
+class CustomItem(Item):
+
+    name = Field()
+
+    def __str__(self):
+        return "name: %s" % self['name']
+
+
 class LoggingContribTest(unittest.TestCase):
 
     def setUp(self):
@@ -27,7 +36,16 @@ class LoggingContribTest(unittest.TestCase):
         exception = Exception(u"\u2018")
         response = Response("http://www.example.com")
         lines = self.formatter.dropped(item, exception, response, self.spider).splitlines()
+        assert all(isinstance(x, unicode) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
 
+    def test_scraped(self):
+        item = CustomItem()
+        item['name'] = u'\xa3'
+        response = Response("http://www.example.com")
+        lines = self.formatter.scraped(item, response, self.spider).splitlines()
+        assert all(isinstance(x, unicode) for x in lines)
+        self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
+
 if __name__ == "__main__":
     unittest.main()