diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 1ea377731..a1b015bf1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -405,8 +405,14 @@ scraped so far, the final code for our Spider would be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in - [dmoz] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in + [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> + {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n], + 'link': [u'http://gnosis.cx/TPiP/'], + 'title': [u'Text Processing in Python']} + [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> + {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], + 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], + 'title': [u'XML Processing with Python']} Storing the scraped data ======================== diff --git a/scrapy/item.py b/scrapy/item.py index 702ee8261..2e8087f98 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,6 +4,7 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from pprint import pformat from UserDict import DictMixin from scrapy.utils.trackref import object_ref @@ -72,11 +73,7 @@ class DictItem(DictMixin, BaseItem): return self._values.keys() def __repr__(self): - """Generate a representation of this item that can be used to - reconstruct the item by evaluating it - """ - values = ', '.join('%s=%r' % field for field in self.iteritems()) - return "%s(%s)" % (self.__class__.__name__, values) + return pformat(dict(self)) class Item(DictItem): diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4fd16c47c..1b5efbd6c 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,3 +1,4 @@ +import os class LogFormatter(object): """Class for generating log messages for different actions. All methods @@ -12,7 +13,7 @@ class LogFormatter(object): request, referer, flags) def scraped(self, item, response, spider): - return "Scraped %s in <%s>" % (item, response.url) + return "Scraped from %s" % response + os.linesep + str(item) def dropped(self, item, exception, response, spider): - return "Dropped %s - %s" % (item, unicode(exception)) + return "Dropped: %s" % unicode(exception) + os.linesep + str(item) diff --git a/scrapy/tests/test_command_shell.py b/scrapy/tests/test_command_shell.py index ffba30415..068492503 100644 --- a/scrapy/tests/test_command_shell.py +++ b/scrapy/tests/test_command_shell.py @@ -12,7 +12,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert 'Item' in out + assert '{}' in out @defer.inlineCallbacks def test_response_body(self): diff --git a/scrapy/tests/test_item.py b/scrapy/tests/test_item.py index 5a55fe7c3..79cac5fc8 100644 --- a/scrapy/tests/test_item.py +++ b/scrapy/tests/test_item.py @@ -50,7 +50,7 @@ class ItemTest(unittest.TestCase): i['number'] = 123 itemrepr = repr(i) self.assertEqual(itemrepr, - "TestItem(name=u'John Doe', number=123)") + "{'name': u'John Doe', 'number': 123}") i2 = eval(itemrepr) self.assertEqual(i2['name'], 'John Doe') diff --git a/scrapy/tests/test_logformatter.py b/scrapy/tests/test_logformatter.py index cfa608f3e..8787bab8a 100644 --- a/scrapy/tests/test_logformatter.py +++ b/scrapy/tests/test_logformatter.py @@ -26,8 +26,8 @@ class LoggingContribTest(unittest.TestCase): item = {} exception = Exception(u"\u2018") response = Response("http://www.example.com") - self.assertEqual(self.formatter.dropped(item, exception, response, self.spider), - u"Dropped {} - \u2018") + lines = self.formatter.dropped(item, exception, response, self.spider).splitlines() + self.assertEqual(lines, [u"Dropped: \u2018", '{}']) if __name__ == "__main__": unittest.main()