diff --git a/scrapy/trunk/scrapy/contrib/adaptors/extraction.py b/scrapy/trunk/scrapy/contrib/adaptors/extraction.py index c059bc350..ef36d0dfe 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/extraction.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/extraction.py @@ -3,6 +3,7 @@ Adaptors related with extraction of data """ import urlparse +from scrapy.http import Response from scrapy.utils.python import flatten from scrapy.xpath.selector import XPathSelector, XPathSelectorList @@ -50,8 +51,7 @@ class ExtractImages(object): Output: list of unicodes """ def __init__(self, response=None, base_url=None): - self.response = response - self.base_url = base_url + self.base_url = response.url if response else base_url def extract_from_xpath(self, selector): ret = [] @@ -75,12 +75,9 @@ class ExtractImages(object): return ret - def __call__(self, locations): - if isinstance(locations, tuple) and len(locations) > 1: - self.base_url = locations[1] - locations = locations[0] - - if not self.response and not self.base_url: + def __call__(self, (locations, base_url)): + self.base_url = base_url.url if isinstance(base_url, Response) else base_url + if not self.base_url: raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.') rel_links = [] @@ -90,6 +87,4 @@ class ExtractImages(object): else: rel_links.append(location) rel_links = extract(rel_links) - - base_url = self.base_url or self.response.url - return [urlparse.urljoin(base_url, link) for link in rel_links] + return [urlparse.urljoin(self.base_url, link) for link in rel_links] diff --git a/scrapy/trunk/scrapy/contrib/pipeline/setguid.py b/scrapy/trunk/scrapy/contrib/pipeline/setguid.py new file mode 100644 index 000000000..275ce8aca --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pipeline/setguid.py @@ -0,0 +1,20 @@ +""" +This pipeline sets the guid for each scraped item. +It should be the previous to the validation pipeline (which is the last one). +""" + +from pydispatch import dispatcher +from scrapy.core import signals + +class SetGUIDPipeline(object): + def __init__(self): + self.spider = None + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) + + def domain_opened(self, domain, spider): + self.spider = spider + + def process_item(self, domain, response, item): + self.spider.set_guid(item) + return item + diff --git a/scrapy/trunk/scrapy/contrib/spiders.py b/scrapy/trunk/scrapy/contrib/spiders.py index e03a86cde..9977d38d5 100644 --- a/scrapy/trunk/scrapy/contrib/spiders.py +++ b/scrapy/trunk/scrapy/contrib/spiders.py @@ -6,13 +6,19 @@ basic crawling. from scrapy.http import Request from scrapy.spider import BaseSpider from scrapy.core.exceptions import UsageError +from scrapy.utils.misc import hash_values class BasicSpider(BaseSpider): - """BasicSpider extends BaseSpider by providing support for simple crawling + """ + BasicSpider extends BaseSpider by providing support for simple crawling by following links contained in web pages. With BasicSpider you can write a basic spider very easily and quickly. For - more information refer to the Scrapy tutorial""" + more information refer to the Scrapy tutorial + """ + + gen_guid_attribs = ['supplier', 'site_id'] + gen_variant_guid_attribs = ['site_id'] def __init__(self): super(BaseSpider, self).__init__() @@ -23,11 +29,7 @@ class BasicSpider(BaseSpider): if attr.startswith('links_'): suffix = attr.split('_', 1)[1] value = getattr(self, attr) - try: - callback = getattr(self, 'parse_%s' % suffix) - except AttributeError: - raise UsageError("%s defines links_%s but doesn't provide a parse_%s method" % \ - (type(self).__name__, suffix, suffix)) + callback = getattr(self, 'parse_%s' % suffix, None) self._links_callback.append((value, callback)) def parse(self, response): @@ -47,7 +49,7 @@ class BasicSpider(BaseSpider): for url, link_text in lx.extract_urls(response).iteritems(): links_to_follow[url] = (callback, link_text) - for url, cb_link in links_to_follow.iteritems(): + for url, (callback, link_text) in links_to_follow.iteritems(): request = Request(url=url, link_text=link_text) request.append_callback(self._parse_wrapper, callback) res.append(request) @@ -55,6 +57,9 @@ class BasicSpider(BaseSpider): def _parse_wrapper(self, response, callback): res = self._links_to_follow(response) - res += callback(response) or () + res += callback(response) if callback else () return res + def set_guid(self, item): + item.guid = hash_values(*[str(getattr(item, aname) or '') for aname in self.gen_guid_attribs]) + diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index f3cc378f8..b7d582c34 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -2,6 +2,7 @@ Auxiliary functions which doesn't fit anywhere else """ import re +import sha from twisted.internet import defer @@ -95,3 +96,20 @@ def extract_regex(regex, text, encoding): return [remove_entities(s, keep=['lt', 'amp']) for s in strings] else: return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] + +def hash_values(*values): + """Hash a series of values. + + For example: + >>> hash_values('some', 'values', 'to', 'hash') + 'f37f5dc65beaaea35af05e16e26d439fd150c576' + """ + hash = sha.new() + for value in values: + if value is None: + message = "hash_values was passed None at argument index %d. This is a bug in the calling code" \ + % list(values).index(None) + raise UsageError(message) + hash.update(value) + return hash.hexdigest() +