From 4ec69bf1332ef6e2d02e073f105800c0a79b1684 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 11 Dec 2008 13:24:33 +0000 Subject: [PATCH] Added logging to the ShoveItem pipeline --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40485 --- .../scrapy/contrib/pipeline/shoveitem.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py b/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py index e4050bf88..f1d5f1773 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py @@ -1,7 +1,7 @@ """ A pipeline to persist objects using shove. -New is a "new generation" shelve. For more information see: +Shove is a "new generation" shelve. For more information see: http://pypi.python.org/pypi/shove """ @@ -10,6 +10,7 @@ from string import Template from shove import Shove from pydispatch import dispatcher +from scrapy import log from scrapy.core import signals from scrapy.conf import settings @@ -26,7 +27,18 @@ class ShoveItemPipeline(object): dispatcher.connect(self.domain_closed, signal=signals.domain_closed) def process_item(self, domain, response, item): - self.stores[domain][str(item.guid)] = item + guid = str(item.guid) + + if guid in self.stores[domain]: + if self.stores[domain][guid] == item: + status = 'old' + else: + status = 'upd' + else: + status = 'new' + + self.stores[domain][guid] = item + self.log(domain, item, status) return item def domain_open(self, domain): @@ -35,3 +47,6 @@ class ShoveItemPipeline(object): def domain_closed(self, domain): self.stores[domain].sync() + + def log(self, domain, item, status): + log.msg("Shove (%s): Item guid=%s" % (status, item.guid), level=log.DEBUG, domain=domain)