From 0ed5978abf0e1d00856ae3c8a64b0000aa9fd89d Mon Sep 17 00:00:00 2001 From: elpolilla Date: Mon, 13 Oct 2008 14:30:51 +0000 Subject: [PATCH] Improved adaptors functionality: - Added many basic adaptors (like extract, extract_links, regex, etc.) - Added some basic pipelines (for single data, lists, urls, etc.) - Now XPathSelectors store the response with which they were created (if any) --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40315 --- scrapy/trunk/scrapy/item/adaptors.py | 211 +++++++++++++++++++++++++- scrapy/trunk/scrapy/item/models.py | 21 +-- scrapy/trunk/scrapy/xpath/selector.py | 6 +- 3 files changed, 224 insertions(+), 14 deletions(-) diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index c95726d90..bf57f5787 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -1,13 +1,29 @@ +import re + from traceback import format_exc -class AdaptorPipe: +from scrapy.xpath.selector import XPathSelector, XPathSelectorList +from scrapy.utils.python import unique, flatten +from scrapy.utils.markup import replace_tags, remove_entities +from scrapy.utils.misc import extract_regex - def __init__(self, adaptors_dict=None): +class AdaptorPipe: + def __init__(self, adaptors_pipe=None): """ Receives a dictionary that maps attribute_name to a list of adaptor functions """ - self.pipes = adaptors_dict or {} + self.pipes = adaptors_pipe or {} + def append_adaptor(self, attrname, adaptor): + """ + Add an adaptor at the end of the provided attribute's pipeline + """ + if callable(adaptor): + if self.pipes.get(attrname): + self.pipes[attrname].append(adaptor) + else: + self.pipes[attrname] = [adaptor] + def execute(self, attrname, value, debug=False): """ Execute pipeline for attribute name "attrname" and value "value". @@ -26,3 +42,192 @@ class AdaptorPipe: return return value + + +############ +# Adaptors # +############ +def extract(location): + """ + This adaptor extracts a list of strings + from 'location', which can be either a list (or tuple), + or an XPathSelector. + + This function *always* returns a list. + """ + if not location: + return [] + elif isinstance(location, (list, tuple)): + return flatten([extract(o) for o in location]) + elif isinstance(location, XPathSelector): + return location.extract() + elif isinstance(location, str): + return [location] + +def _absolutize_links(rel_links, current_url, base_url): + abs_links = [] + for link in rel_links: + if link.startswith('/'): + abs_links.append('%s%s' % (base_url, link)) + elif link.startswith('http://'): + abs_links.append(link) + else: + abs_links.append('%s/%s' % (current_url, link)) + return abs_links + +def extract_links(locations): + """ + This adaptor receives either an XPathSelector containing + the desired locations for finding urls, or a tuple like (xpath, regexp) + containing the xpath locations to look in, and a regular expression + to parse those locations. + + In any case, this adaptor returns a list of absolute urls extracted. + """ + ret = [] + if locations: + regexp = None + if isinstance(locations, XPathSelector): + locations = XPathSelectorList([locations]) + elif isinstance(locations, tuple): + locations, regexp = locations + + if isinstance(locations, XPathSelectorList): + if regexp: + ret = locations.re(regexp) + else: + for selector in locations: + if selector.xmlNode.type == 'element': + if selector.xmlNode.name == 'a': + children = selector.x('child::*') + if len(children) > 1: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + else: + ret.extend(selector.x('@href')) + elif selector.xmlNode.name == 'img': + ret.extend(selector.x('@src')) + else: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: + ret.append(selector) + ret = [selector.extract() for selector in ret] + current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)/', locations[0].response.url).groups() + ret = _absolutize_links(ret, current_url, base_url) + return ret + +def to_unicode(value): + """ + Receives a list of strings, converts + it to unicode, and returns a new list. + """ + if isinstance(value, (list, tuple)): + return [ unicode(v) for v in value ] + else: + raise TypeError('to_unicode adaptor must receive a list or a tuple.') + +def regex(expr): + """ + This factory function returns a ready-to-use + adaptor for the specified regular expression. + This adaptor will accept either an XPathSelectorList + or a list of strings, and will apply the provided regular + expression to each of its members. + + This adaptor always returns a list of strings. + """ + def _regex(value): + if isinstance(value, (XPathSelector, XPathSelectorList)): + return value.re(expr) + elif isinstance(value, list) and value: + return extract_regex(expr, value, 'utf-8') + return value + return _regex + +def unquote_all(value): + """ + Receives a list of strings, removes all of the + entities the strings may have, and returns + a new list + """ + return [ remove_entities(v) for v in value ] + +def unquote(value): + """ + Receives a list of strings, removes all of the entities + the strings may have (except for < and &), and + returns a new list + """ + return [ remove_entities(v, keep=['lt', 'amp']) for v in value ] + +def remove_tags(value): + return [ replace_tags(v) for v in value ] + +_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) +def _remove_root(value): + m = _remove_root_re.search(value) + if m: + value = m.group(1) + return value + +def remove_root(value): + return [ _remove_root(v) for v in value ] + +_clean_spaces_re = re.compile("\s+", re.U) +def remove_multispaces(value): + return [ _clean_spaces_re.sub(' ', v) for v in value ] + +def strip(value): + return [ v.strip() for v in value ] + +def drop_empty_elements(value): + return [ v for v in value if v ] + +def delist(value): + return ' '.join(value) + + + +############# +# Pipelines # +############# +""" +The following methods automatically generate adaptor pipelines +for some basic datatypes, according to the parameters you pass them. +""" +def single_pipeline(do_unquote=True): + pipe = [ extract, + unique, + to_unicode, + drop_empty_elements, + remove_tags, + remove_root, + remove_multispaces, + strip, + ] + if do_unquote: + pipe.append(unquote) + return pipe + [delist] + +def url_pipeline(): + return [ extract_links, + unique, + to_unicode, + drop_empty_elements, + ] + +def list_pipeline(): + return [ extract, + unique, + to_unicode, + drop_empty_elements, + unquote, + remove_tags, + remove_root, + strip, + ] + +def list_join_pipeline(delimiter='\t'): + return list_pipeline() + [delimiter.join] + \ No newline at end of file diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 96ac1d9ed..a68568104 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -7,15 +7,18 @@ class ScrapedItem(object): that identifies uniquely the given scraped item. """ - def setadaptors(self, adaptors_dict): - """Set adaptors to use for this item. Receives a dict of adaptors and - returns the item itself""" - self.adaptors_dict = AdaptorPipe(adaptors_dict) + def setadaptors(self, adaptors_pipe): + """ + Set adaptors to use for this item. Receives a dict of adaptors and + returns the item itself. + """ + object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe)) return self - + + def append_adaptor(self, attrname, adaptor): + self._adaptors_pipe.append_adaptor(attrname, adaptor) + def attribute(self, attrname, value): - value = self.adaptors_pipe.execute(attrname, value) - if not hasattr(self, attrname): + value = self._adaptors_pipe.execute(attrname, value) + if not hasattr(self, attrname) or not getattr(self, attrname): setattr(self, attrname, value) - - diff --git a/scrapy/trunk/scrapy/xpath/selector.py b/scrapy/trunk/scrapy/xpath/selector.py index 4709da696..cb97f7da8 100644 --- a/scrapy/trunk/scrapy/xpath/selector.py +++ b/scrapy/trunk/scrapy/xpath/selector.py @@ -32,6 +32,7 @@ class XPathSelector(object): self.doc = Libxml2Document(response, constructor=constructor) self.xmlNode = self.doc.xmlDoc self.expr = expr + self.response = response def x(self, xpath): """Perform the given XPath query on the current XPathSelector and @@ -41,9 +42,10 @@ class XPathSelector(object): xpath_result = self.doc.xpathContext.xpathEval(xpath) cls = type(self) if hasattr(xpath_result, '__iter__'): - return XPathSelectorList([cls(node=node, parent=self, expr=xpath) for node in xpath_result]) + return XPathSelectorList([cls(node=node, parent=self, expr=xpath, response=self.response) + for node in xpath_result]) else: - return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath)]) + return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)]) else: return XPathSelectorList([])