From f2bab509796ddf13495a1e17c5b8a3846fec979d Mon Sep 17 00:00:00 2001
From: elpolilla <none@none>
Date: Mon, 3 Nov 2008 11:02:58 +0000
Subject: [PATCH] - Fixed bad implementation of the SetGUIDPipeline - Modified
 item's attribute method to have an optional 'add' argument - Renamed
 normalize_urls adaptor to canonicalize_urls

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40355
---
 scrapy/trunk/scrapy/contrib/adaptors/__init__.py |  2 +-
 scrapy/trunk/scrapy/contrib/adaptors/misc.py     |  2 +-
 scrapy/trunk/scrapy/contrib/item/models.py       | 10 ++--------
 scrapy/trunk/scrapy/item/adaptors.py             |  3 +++
 scrapy/trunk/scrapy/item/models.py               | 13 +++++++++++--
 scrapy/trunk/scrapy/xpath/selector.py            |  1 +
 6 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
index 97e76e48b..b3951c0fc 100644
--- a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
+++ b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
@@ -1,4 +1,4 @@
 from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
 from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
-from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, Delist, Regex
+from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, Delist, Regex
 from scrapy.utils.python import unique, flatten
diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py
index 06bd16d61..1453ff0aa 100644
--- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py
+++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py
@@ -27,7 +27,7 @@ def strip_list(value):
 def drop_empty(value):
     return [ v for v in value if v ]
 
-def normalize_urls(value):
+def canonicalize_urls(value):
     if hasattr(value, '__iter__'):
         return [canonicalize_url(url) for url in value]
     elif isinstance(value, basestring):
diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py
index 378ca208a..65ff653de 100644
--- a/scrapy/trunk/scrapy/contrib/item/models.py
+++ b/scrapy/trunk/scrapy/contrib/item/models.py
@@ -9,6 +9,7 @@ from pydispatch import dispatcher
 from pprint import PrettyPrinter
 
 from scrapy.item import ScrapedItem, ItemDelta
+from scrapy.spider import spiders
 from scrapy.core import signals
 from scrapy.core.exceptions import UsageError, DropItem
 
@@ -30,15 +31,8 @@ class ValidationPipeline(object):
         return item
 
 class SetGUIDPipeline(object):
-    def __init__(self):
-        self.spider = None
-        dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
-
-    def domain_opened(self, domain, spider):
-        self.spider = spider
-
     def process_item(self, domain, response, item):
-        self.spider.set_guid(item)
+        spiders.fromdomain(domain).set_guid(item)
         return item
 
 class RobustScrapedItem(ScrapedItem):
diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py
index 2580c955b..3e366cc63 100644
--- a/scrapy/trunk/scrapy/item/adaptors.py
+++ b/scrapy/trunk/scrapy/item/adaptors.py
@@ -31,3 +31,6 @@ class AdaptorDict(dict):
         
         return value
 
+
+    def __repr__(self):
+        return "<AdaptorDict [ %d attributes ] >" % len(self.keys())
diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py
index 8a5bfe2c9..6a23a4a02 100644
--- a/scrapy/trunk/scrapy/item/models.py
+++ b/scrapy/trunk/scrapy/item/models.py
@@ -22,10 +22,19 @@ class ScrapedItem(object):
         """
         self._adaptors_dict[attrib] = adaptors
 
-    def attribute(self, attrname, value, debug=False, override=False):
+    def attribute(self, attrname, value, override=False, add=False, debug=False):
         val = self._adaptors_dict.execute(attrname, value, debug)
-        if not getattr(self, attrname, None) or override:
+        curr_val = getattr(self, attrname, None)
+        if not curr_val:
             setattr(self, attrname, val)
+        else:
+            if override:
+                setattr(self, attrname, val)
+            elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)):
+                newval = []
+                newval.extend(curr_val)
+                newval.extend(val)
+                setattr(self, attrname, newval)
 
     def __sub__(self, other):
         raise NotImplementedError
diff --git a/scrapy/trunk/scrapy/xpath/selector.py b/scrapy/trunk/scrapy/xpath/selector.py
index cb97f7da8..7ddd108dc 100644
--- a/scrapy/trunk/scrapy/xpath/selector.py
+++ b/scrapy/trunk/scrapy/xpath/selector.py
@@ -48,6 +48,7 @@ class XPathSelector(object):
                 return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)])
         else:
             return XPathSelectorList([])
+    __call__ = x
 
     def re(self, regex):
         """Return a list of unicode strings by applying the regex over all