From 38c3f7d0b455b07b1904778aa3f03d47ee775ab5 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 23 Jul 2009 11:49:48 -0300 Subject: [PATCH] Some changes to logging of scraped items: 1. "Scraped Item" log level changed to DEBUG 2. "Dropped Item" log level changed to WARNING 3. added "Passed Item" log message with INFO level --- docs/intro/tutorial.rst | 4 ++-- scrapy/core/scraper.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6a8ba3c95..f860ccedf 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -389,8 +389,8 @@ should be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz.org] INFO: Scraped DmozItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in - [dmoz.org] INFO: Scraped DmozItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in + [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in + [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in Storing the data (using an Item Pipeline) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2be14fad9..eac4176cf 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -152,7 +152,8 @@ class Scraper(object): spider=spider) self.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): - log.msg("Scraped %s in <%s>" % (output, request.url), domain=domain) + log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \ + domain=domain) signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \ item=output, spider=spider, response=response) self.sites[domain].itemproc_size += 1 @@ -187,13 +188,14 @@ class Scraper(object): if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - log.msg("Dropped %s - %s" % (item, str(ex)), log.DEBUG, domain=domain) + log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, domain=domain) signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, \ item=item, spider=spider, exception=output.value) else: log.msg('Error processing %s - %s' % (item, output), \ log.ERROR, domain=domain) else: + log.msg("Passed %s" % item, log.INFO, domain=domain) signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, \ item=item, spider=spider, output=output)