From 5bf733b6f624cc5a3ddf8a7e46337dd3290740b2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 3 Jun 2011 01:13:01 -0300 Subject: [PATCH] Changed default representation of items to pretty-printed dicts. This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines. Projects can still customize how items are represented by overriding the item's __str__ method, as usual. --- docs/intro/tutorial.rst | 10 ++++++++-- scrapy/item.py | 7 ++----- scrapy/logformatter.py | 5 +++-- scrapy/tests/test_command_shell.py | 2 +- scrapy/tests/test_item.py | 2 +- scrapy/tests/test_logformatter.py | 4 ++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 1ea377731..a1b015bf1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -405,8 +405,14 @@ scraped so far, the final code for our Spider would be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in - [dmoz] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in + [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> + {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n], + 'link': [u'http://gnosis.cx/TPiP/'], + 'title': [u'Text Processing in Python']} + [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> + {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], + 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], + 'title': [u'XML Processing with Python']} Storing the scraped data ======================== diff --git a/scrapy/item.py b/scrapy/item.py index 702ee8261..2e8087f98 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,6 +4,7 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from pprint import pformat from UserDict import DictMixin from scrapy.utils.trackref import object_ref @@ -72,11 +73,7 @@ class DictItem(DictMixin, BaseItem): return self._values.keys() def __repr__(self): - """Generate a representation of this item that can be used to - reconstruct the item by evaluating it - """ - values = ', '.join('%s=%r' % field for field in self.iteritems()) - return "%s(%s)" % (self.__class__.__name__, values) + return pformat(dict(self)) class Item(DictItem): diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4fd16c47c..1b5efbd6c 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,3 +1,4 @@ +import os class LogFormatter(object): """Class for generating log messages for different actions. All methods @@ -12,7 +13,7 @@ class LogFormatter(object): request, referer, flags) def scraped(self, item, response, spider): - return "Scraped %s in <%s>" % (item, response.url) + return "Scraped from %s" % response + os.linesep + str(item) def dropped(self, item, exception, response, spider): - return "Dropped %s - %s" % (item, unicode(exception)) + return "Dropped: %s" % unicode(exception) + os.linesep + str(item) diff --git a/scrapy/tests/test_command_shell.py b/scrapy/tests/test_command_shell.py index ffba30415..068492503 100644 --- a/scrapy/tests/test_command_shell.py +++ b/scrapy/tests/test_command_shell.py @@ -12,7 +12,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert 'Item' in out + assert '{}' in out @defer.inlineCallbacks def test_response_body(self): diff --git a/scrapy/tests/test_item.py b/scrapy/tests/test_item.py index 5a55fe7c3..79cac5fc8 100644 --- a/scrapy/tests/test_item.py +++ b/scrapy/tests/test_item.py @@ -50,7 +50,7 @@ class ItemTest(unittest.TestCase): i['number'] = 123 itemrepr = repr(i) self.assertEqual(itemrepr, - "TestItem(name=u'John Doe', number=123)") + "{'name': u'John Doe', 'number': 123}") i2 = eval(itemrepr) self.assertEqual(i2['name'], 'John Doe') diff --git a/scrapy/tests/test_logformatter.py b/scrapy/tests/test_logformatter.py index cfa608f3e..8787bab8a 100644 --- a/scrapy/tests/test_logformatter.py +++ b/scrapy/tests/test_logformatter.py @@ -26,8 +26,8 @@ class LoggingContribTest(unittest.TestCase): item = {} exception = Exception(u"\u2018") response = Response("http://www.example.com") - self.assertEqual(self.formatter.dropped(item, exception, response, self.spider), - u"Dropped {} - \u2018") + lines = self.formatter.dropped(item, exception, response, self.spider).splitlines() + self.assertEqual(lines, [u"Dropped: \u2018", '{}']) if __name__ == "__main__": unittest.main()