- Added SetGUIDPipeline and the guid generation helper for the BasicSpider

- Fixed some issues with BasicSpider
- Added a normalize_url adaptor

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40348

scrapy/trunk/scrapy/contrib/adaptors/extraction.py

@@ -3,6 +3,7 @@ Adaptors related with extraction of data
 """
 
 import urlparse
+from scrapy.http import Response
 from scrapy.utils.python import flatten
 from scrapy.xpath.selector import XPathSelector, XPathSelectorList
 
@@ -50,8 +51,7 @@ class ExtractImages(object):
     Output: list of unicodes
     """
     def __init__(self, response=None, base_url=None):
-        self.response = response
-        self.base_url = base_url
+        self.base_url = response.url if response else base_url
 
     def extract_from_xpath(self, selector):
         ret = []
@@ -75,12 +75,9 @@ class ExtractImages(object):
 
         return ret
 
-    def __call__(self, locations):
-        if isinstance(locations, tuple) and len(locations) > 1:
-            self.base_url = locations[1]
-            locations = locations[0]
-
-        if not self.response and not self.base_url:
+    def __call__(self, (locations, base_url)):
+        self.base_url = base_url.url if isinstance(base_url, Response) else base_url
+        if not self.base_url:
             raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.')
         
         rel_links = []
@@ -90,6 +87,4 @@ class ExtractImages(object):
             else:
                 rel_links.append(location)
         rel_links = extract(rel_links)
-
-        base_url = self.base_url or self.response.url
-        return [urlparse.urljoin(base_url, link) for link in rel_links]
+        return [urlparse.urljoin(self.base_url, link) for link in rel_links]

scrapy/trunk/scrapy/contrib/pipeline/setguid.py

@@ -0,0 +1,20 @@
+"""
+This pipeline sets the guid for each scraped item.
+It should be the previous to the validation pipeline (which is the last one).
+"""
+
+from pydispatch import dispatcher
+from scrapy.core import signals
+
+class SetGUIDPipeline(object):
+    def __init__(self):
+        self.spider = None
+        dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
+
+    def domain_opened(self, domain, spider):
+        self.spider = spider
+
+    def process_item(self, domain, response, item):
+        self.spider.set_guid(item)
+        return item
+

scrapy/trunk/scrapy/contrib/spiders.py

@@ -6,13 +6,19 @@ basic crawling.
 from scrapy.http import Request
 from scrapy.spider import BaseSpider
 from scrapy.core.exceptions import UsageError
+from scrapy.utils.misc import hash_values
 
 class BasicSpider(BaseSpider):
-    """BasicSpider extends BaseSpider by providing support for simple crawling
+    """
+    BasicSpider extends BaseSpider by providing support for simple crawling
     by following links contained in web pages. 
     
     With BasicSpider you can write a basic spider very easily and quickly. For
-    more information refer to the Scrapy tutorial"""
+    more information refer to the Scrapy tutorial
+    """
+
+    gen_guid_attribs = ['supplier', 'site_id']
+    gen_variant_guid_attribs = ['site_id']
 
     def __init__(self):
         super(BaseSpider, self).__init__()
@@ -23,11 +29,7 @@ class BasicSpider(BaseSpider):
             if attr.startswith('links_'):
                 suffix = attr.split('_', 1)[1]
                 value = getattr(self, attr)
-                try:
-                    callback = getattr(self, 'parse_%s' % suffix)
-                except AttributeError:
-                    raise UsageError("%s defines links_%s but doesn't provide a parse_%s method" % \
-                                     (type(self).__name__, suffix, suffix))
+                callback = getattr(self, 'parse_%s' % suffix, None)
                 self._links_callback.append((value, callback))
 
     def parse(self, response):
@@ -47,7 +49,7 @@ class BasicSpider(BaseSpider):
             for url, link_text in lx.extract_urls(response).iteritems():
                 links_to_follow[url] = (callback, link_text)
 
-        for url, cb_link in links_to_follow.iteritems():
+        for url, (callback, link_text) in links_to_follow.iteritems():
             request = Request(url=url, link_text=link_text)
             request.append_callback(self._parse_wrapper, callback)
             res.append(request)
@@ -55,6 +57,9 @@ class BasicSpider(BaseSpider):
 
     def _parse_wrapper(self, response, callback):
         res = self._links_to_follow(response)
-        res += callback(response) or ()
+        res += callback(response) if callback else ()
         return res
 
+    def set_guid(self, item):
+        item.guid = hash_values(*[str(getattr(item, aname) or '') for aname in self.gen_guid_attribs])
+

scrapy/trunk/scrapy/utils/misc.py

@@ -2,6 +2,7 @@
 Auxiliary functions which doesn't fit anywhere else
 """
 import re
+import sha
 
 from twisted.internet import defer
 
@@ -95,3 +96,20 @@ def extract_regex(regex, text, encoding):
         return [remove_entities(s, keep=['lt', 'amp']) for s in strings]
     else:
         return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings]
+
+def hash_values(*values):
+    """Hash a series of values. 
+    
+    For example:
+    >>> hash_values('some', 'values', 'to', 'hash')
+    'f37f5dc65beaaea35af05e16e26d439fd150c576'
+    """
+    hash = sha.new()
+    for value in values:
+        if value is None:
+            message = "hash_values was passed None at argument index %d. This is a bug in the calling code" \
+                    % list(values).index(None)
+            raise UsageError(message)
+        hash.update(value)
+    return hash.hexdigest()
+