- Fixed bad implementation of the SetGUIDPipeline

- Modified item's attribute method to have an optional 'add' argument
- Renamed normalize_urls adaptor to canonicalize_urls

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40355

scrapy/trunk/scrapy/contrib/adaptors/__init__.py

@@ -1,4 +1,4 @@
 from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
 from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
-from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, Delist, Regex
+from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, Delist, Regex
 from scrapy.utils.python import unique, flatten

scrapy/trunk/scrapy/contrib/adaptors/misc.py

@@ -27,7 +27,7 @@ def strip_list(value):
 def drop_empty(value):
     return [ v for v in value if v ]
 
-def normalize_urls(value):
+def canonicalize_urls(value):
     if hasattr(value, '__iter__'):
         return [canonicalize_url(url) for url in value]
     elif isinstance(value, basestring):

scrapy/trunk/scrapy/contrib/item/models.py

@@ -9,6 +9,7 @@ from pydispatch import dispatcher
 from pprint import PrettyPrinter
 
 from scrapy.item import ScrapedItem, ItemDelta
+from scrapy.spider import spiders
 from scrapy.core import signals
 from scrapy.core.exceptions import UsageError, DropItem
 
@@ -30,15 +31,8 @@ class ValidationPipeline(object):
         return item
 
 class SetGUIDPipeline(object):
-    def __init__(self):
-        self.spider = None
-        dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
-
-    def domain_opened(self, domain, spider):
-        self.spider = spider
-
     def process_item(self, domain, response, item):
-        self.spider.set_guid(item)
+        spiders.fromdomain(domain).set_guid(item)
         return item
 
 class RobustScrapedItem(ScrapedItem):

scrapy/trunk/scrapy/item/adaptors.py

@@ -31,3 +31,6 @@ class AdaptorDict(dict):
         
         return value
 
+
+    def __repr__(self):
+        return "<AdaptorDict [ %d attributes ] >" % len(self.keys())

scrapy/trunk/scrapy/item/models.py

@@ -22,10 +22,19 @@ class ScrapedItem(object):
         """
         self._adaptors_dict[attrib] = adaptors
 
-    def attribute(self, attrname, value, debug=False, override=False):
+    def attribute(self, attrname, value, override=False, add=False, debug=False):
         val = self._adaptors_dict.execute(attrname, value, debug)
-        if not getattr(self, attrname, None) or override:
+        curr_val = getattr(self, attrname, None)
+        if not curr_val:
             setattr(self, attrname, val)
+        else:
+            if override:
+                setattr(self, attrname, val)
+            elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)):
+                newval = []
+                newval.extend(curr_val)
+                newval.extend(val)
+                setattr(self, attrname, newval)
 
     def __sub__(self, other):
         raise NotImplementedError

scrapy/trunk/scrapy/xpath/selector.py

@@ -48,6 +48,7 @@ class XPathSelector(object):
                 return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)])
         else:
             return XPathSelectorList([])
+    __call__ = x
 
     def re(self, regex):
         """Return a list of unicode strings by applying the regex over all