From 42e6ed74e501b5c69982abcc78a10e8a51119744 Mon Sep 17 00:00:00 2001
From: elpolilla <none@none>
Date: Fri, 24 Oct 2008 00:29:27 +0000
Subject: [PATCH] Modified some code to avoid problems with the _adaptor_pipe
 attribute and the replays, or with the item themselves

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40340
---
 scrapy/trunk/scrapy/contrib/item/models.py | 15 +++++++---
 scrapy/trunk/scrapy/item/__init__.py       |  2 +-
 scrapy/trunk/scrapy/replay/__init__.py     | 34 +++++++++++++++++-----
 3 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py
index 451d90b2a..0f8679c47 100644
--- a/scrapy/trunk/scrapy/contrib/item/models.py
+++ b/scrapy/trunk/scrapy/contrib/item/models.py
@@ -6,7 +6,7 @@ useful in some Scrapy implementations
 import hashlib
 
 from pprint import PrettyPrinter
-from scrapy.item import ScrapedItem
+from scrapy.item import ScrapedItem, ItemDelta
 from scrapy.core.exceptions import UsageError, DropItem
 
 class ValidationError(DropItem):
@@ -37,11 +37,12 @@ class RobustScrapedItem(ScrapedItem):
         'url': basestring,  # the main URL where this item was scraped from
     }
     
-    def __init__(self, data=None):
+    def __init__(self, data=None, adaptors_pipe={}):
         """
         A scraped item can be initialised with a dictionary that will be
         squirted directly into the object.
         """
+        super(RobustScrapedItem, self).__init__(adaptors_pipe)
         if isinstance(data, dict):
             for attr, value in data.iteritems():
                 setattr(self, attr, value)
@@ -70,6 +71,9 @@ class RobustScrapedItem(ScrapedItem):
             self.__dict__.pop(attr, None)
             return
 
+        if attr == '_adaptors_pipe':
+            return object.__setattr__(self, '_adaptors_pipe', value)
+
         type1 = self.ATTRIBUTES[attr]
         if hasattr(type1, '__iter__'):
             if not hasattr(value, '__iter__'):
@@ -138,14 +142,14 @@ class RobustScrapedItem(ScrapedItem):
 
         item.features.append('feature')
         """
-        if self._version:
+        if getattr(self, '_version', None):
             return self._version
         hash_ = hashlib.sha1()
         hash_.update("".join(["".join([n, str(v)]) for n,v in sorted(self.__dict__.iteritems())]))
         return hash_.hexdigest()
 
 
-class RobustItemDelta(object):
+class RobustItemDelta(ItemDelta):
     """
     This class represents the difference between
     a pair of RobustScrapedItems.
@@ -199,6 +203,9 @@ class RobustItemDelta(object):
                 return True
         return False
 
+    def __nonzero__(self):
+        return bool(self.diff)
+
     def __repr__(self):
         if self.diff:
             pp = PrettyPrinter(indent=3)
diff --git a/scrapy/trunk/scrapy/item/__init__.py b/scrapy/trunk/scrapy/item/__init__.py
index 35982e691..423ebfb01 100644
--- a/scrapy/trunk/scrapy/item/__init__.py
+++ b/scrapy/trunk/scrapy/item/__init__.py
@@ -1 +1 @@
-from scrapy.item.models import ScrapedItem
+from scrapy.item.models import ScrapedItem, ItemDelta
diff --git a/scrapy/trunk/scrapy/replay/__init__.py b/scrapy/trunk/scrapy/replay/__init__.py
index 38512e824..5026914a5 100644
--- a/scrapy/trunk/scrapy/replay/__init__.py
+++ b/scrapy/trunk/scrapy/replay/__init__.py
@@ -5,14 +5,16 @@ import cPickle as pickle
 import shutil
 import tempfile
 import tarfile
+import copy
 
 from pydispatch import dispatcher
 
-from scrapy.core import signals
 from scrapy import log
+from scrapy.core import signals
 from scrapy.core.manager import scrapymanager
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
+from scrapy.item import ScrapedItem
 
 class Replay(object):
     """
@@ -99,17 +101,35 @@ class Replay(object):
         self._save()
         self.cleanup()
 
-    def item_scraped(self, item, spider):
-        if self.recording or self.updating:
-            self.scraped_old[str(item.guid)] = item.copy()
+    def clean_private(self, src):
+        if isinstance(src, ScrapedItem):
+            target = src.__dict__
+        elif isinstance(src, (list, tuple)):
+            return type(src)(self.clean_private(elem) for elem in src)
+        elif isinstance(src, dict):
+            target = src
         else:
-            self.scraped_new[str(item.guid)] = item.copy()
+            return src
+        for key, val in target.items():
+            if key[0] == '_' and key[1] != '_':
+                del target[key]
+            else:
+                target[key] = self.clean_private(val)
+        return src
+
+    def item_scraped(self, item, spider):
+        item = self.clean_private(copy.deepcopy(item))
+        if self.recording or self.updating:
+            self.scraped_old[str(item.guid)] = item
+        else:
+            self.scraped_new[str(item.guid)] = item
 
     def item_passed(self, item, spider):
+        item = self.clean_private(copy.deepcopy(item))
         if self.recording or self.updating:
-            self.passed_old[str(item.guid)] = item.copy()
+            self.passed_old[str(item.guid)] = item
         else:
-            self.passed_new[str(item.guid)] = item.copy()
+            self.passed_new[str(item.guid)] = item
 
     def response_received(self, response, spider):
         #key = response.request.fingerprint()