diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index 17cd58644..67e3243e4 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -1,197 +1,268 @@ import re +import cPickle as pickle from traceback import format_exc +from urlparse import urlparse from scrapy.xpath.selector import XPathSelector, XPathSelectorList from scrapy.utils.python import unique, flatten from scrapy.utils.markup import replace_tags, remove_entities from scrapy.utils.misc import extract_regex +from scrapy.conf import settings +from scrapy import log + +class AdaptorPipe(object): + """ + Class that represents an item's attribute pipeline. + + This class contains a dictionary of attributes, matched with a list + of adaptors to be run for filtering the input before storing. + """ -class AdaptorPipe: def __init__(self, adaptors_pipe=None): """ Receives a dictionary that maps attribute_name to a list of adaptor functions """ self.pipes = adaptors_pipe or {} - def append_adaptor(self, attrname, adaptor): + def set_adaptors(self, attr, adaptors): """ - Add an adaptor at the end of the provided attribute's pipeline + Set the adaptor pipeline that will be used for the specified attribute """ - if callable(adaptor): - if self.pipes.get(attrname): - self.pipes[attrname].append(adaptor) - else: - self.pipes[attrname] = [adaptor] - - def execute(self, attrname, value, debug=False): + self.pipes[attr] = adaptors + + def execute(self, attrname, value, kwargs): """ Execute pipeline for attribute name "attrname" and value "value". """ - for function in self.pipes.get(attrname, []): + debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']) + + for adaptor in self.pipes.get(attrname, []): try: if debug: - print " %07s | input >" % function.func_name, repr(value) - value = function(value) + print " %07s | input >" % adaptor.__name__, repr(value) + value = adaptor(kwargs)(value) if debug: - print " %07s | output>" % function.func_name, repr(value) - + print " %07s | output >" % adaptor.__name__, repr(value) + except Exception, e: - print "Error in '%s' adaptor. Traceback text:" % function.func_name + print "Error in '%s' adaptor. Traceback text:" % adaptor.__name__ print format_exc() return - + return value +class AdaptorFunc(object): + """ + This is the base class for adaptors. + + An adaptor is just an object subclassed from this class + which defines the __call__ method, and receives/returns only + one value. + + You can send the adaptor some extra options while creating it (just before running it) + through **kwargs, and managing them by overriding the __init__ method, as shown on + the UnquoteAdaptor, for example. + """ + def __init__(self, kwargs={}): + pass + + def __call__(self): + raise NotImplementedError('You must define the __call__ method to create and use an adaptor') ############ # Adaptors # ############ -def extract(location): +class ExtractAdaptor(AdaptorFunc): """ This adaptor extracts a list of strings from 'location', which can be either a list (or tuple), or an XPathSelector. - This function *always* returns a list. + This adaptor *always* returns a list. """ - if not location: - return [] - elif isinstance(location, XPathSelectorList): - return flatten([extract(o) for o in location]) - elif isinstance(location, XPathSelector): - return location.extract() - elif isinstance(location, (list, tuple)): - return flatten(location) - elif isinstance(location, basestring): - return [location] -def _absolutize_links(rel_links, current_url, base_url): - abs_links = [] - for link in rel_links: - if link.startswith('/'): - abs_links.append('%s%s' % (base_url, link)) - elif link.startswith('http://'): - abs_links.append(link) - else: - abs_links.append('%s/%s' % (current_url, link)) - return abs_links - -def extract_links(locations): + def __call__(self, location): + if not location: + return [] + elif isinstance(location, (XPathSelector, XPathSelectorList)): + return flatten(location.extract()) + elif isinstance(location, (list, tuple)): + return flatten(map(lambda x: x.extract() if isinstance(x, (XPathSelector, XPathSelectorList)) else x, flatten(location))) + elif isinstance(location, basestring): + return [location] + +class ExtractImagesAdaptor(AdaptorFunc): """ This adaptor receives either an XPathSelector containing the desired locations for finding urls, or a tuple like (xpath, regexp) containing the xpath locations to look in, and a regular expression to parse those locations. - In any case, this adaptor returns a list of absolute urls extracted. + In any case, this adaptor returns a list containing the absolute urls extracted. """ - ret = [] - if locations: - regexp = None - if isinstance(locations, XPathSelector): - locations = XPathSelectorList([locations]) - elif isinstance(locations, tuple): - locations, regexp = locations - - if isinstance(locations, XPathSelectorList): - if regexp: - ret = locations.re(regexp) - else: - for selector in locations: - if selector.xmlNode.type == 'element': - if selector.xmlNode.name == 'a': - children = selector.x('child::*') - if len(children) > 1: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif len(children) == 1 and children[0].xmlNode.name == 'img': - ret.extend(children.x('@src')) - else: - ret.extend(selector.x('@href')) - elif selector.xmlNode.name == 'img': - ret.extend(selector.x('@src')) - else: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: - ret.append(selector) - ret = [selector.extract() for selector in ret] - current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups() - ret = _absolutize_links(ret, current_url, base_url) - return ret -def to_unicode(value): + def __init__(self, kwargs): + self.base_url = kwargs.get('base_url') + self.response = kwargs.get('response') + if not self.response and not self.base_url: + raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.') + + def extract_from_xpath(self, selector): + ret = [] + + if selector.xmlNode.type == 'element': + if selector.xmlNode.name == 'a': + children = selector.x('child::*') + if len(children) > 1: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + elif len(children) == 1 and children[0].xmlNode.name == 'img': + ret.extend(children.x('@src')) + else: + ret.extend(selector.x('@href')) + elif selector.xmlNode.name == 'img': + ret.extend(selector.x('@src')) + else: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: + ret.append(selector) + + return ret + + def absolutize_link(self, base_url, link): + base_url = urlparse(base_url) + ret = [] + + if link.startswith('/'): + ret.append('http://%s%s' % (base_url.hostname, link)) + elif link.startswith('http://'): + ret.append(link) + else: + ret.append('http://%s%s/%s' % (base_url.hostname, base_url.path, link)) + + return ret + + def __call__(self, locations): + rel_links = [] + for location in flatten(locations): + if isinstance(location, (XPathSelector, XPathSelectorList)): + rel_links.extend(self.extract_from_xpath(location)) + else: + rel_links.append(location) + rel_links = ExtractAdaptor()(rel_links) + + if self.response: + return flatten([self.absolutize_link(self.response.url, link) for link in rel_links]) + elif self.base_url: + return flatten([self.absolutize_link(self.base_url, link) for link in rel_links]) + else: + abs_links = [] + for link in rel_links: + if link.startswith('http://'): + abs_links.append(link) + else: + log.msg('Couldnt get the absolute url for "%s". Ignoring link...' % link, 'WARNING') + return abs_links + +class BoolAdaptor(AdaptorFunc): + def __call__(self, value): + return bool(value) + +class ToUnicodeAdaptor(AdaptorFunc): """ Receives a list of strings, converts it to unicode, and returns a new list. """ - if isinstance(value, (list, tuple)): - return [ unicode(v) for v in value ] - else: - raise TypeError('to_unicode adaptor must receive a list or a tuple.') -def regex(expr): - """ - This factory function returns a ready-to-use - adaptor for the specified regular expression. - This adaptor will accept either an XPathSelectorList - or a list of strings, and will apply the provided regular - expression to each of its members. + def __call__(self, value): + if isinstance(value, (list, tuple)): + return [ unicode(v) for v in value ] + else: + raise TypeError('ToUnicodeAdaptor must receive either a list or a tuple.') - This adaptor always returns a list of strings. +class RegexAdaptor(AdaptorFunc): """ - def _regex(value): - if isinstance(value, (XPathSelector, XPathSelectorList)): - return value.re(expr) - elif isinstance(value, list) and value: - return flatten([extract_regex(expr, string, 'utf-8') for string in value]) - return value - return _regex + This adaptor must receive either a list of strings or an XPathSelector + and return a new list with the matches of the given strings with the given regular + expression (which is passed by a keyword argument, and is mandatory for this adaptor). + """ + + def __init__(self, kwargs): + self.regex = kwargs.get('regex') -def unquote_all(value): + def __call__(self, value): + if self.regex: + if isinstance(value, (XPathSelector, XPathSelectorList)): + return value.re(self.regex) + elif isinstance(value, list) and value: + return flatten([extract_regex(self.regex, string, 'utf-8') for string in value]) + return value + +class UnquoteAdaptor(AdaptorFunc): """ Receives a list of strings, removes all of the entities the strings may have, and returns a new list """ - return [ remove_entities(v) for v in value ] -def unquote(value): - """ - Receives a list of strings, removes all of the entities - the strings may have (except for < and &), and - returns a new list - """ - return [ remove_entities(v, keep=['lt', 'amp']) for v in value ] + def __init__(self, kwargs={}): + self.keep = kwargs.get('keep', ['lt', 'amp']) -def remove_tags(value): - return [ replace_tags(v) for v in value ] + def __call__(self, value): + return [ remove_entities(v, keep=self.keep) for v in value ] + +class RemoveTagsAdaptor(AdaptorFunc): + def __call__(self, value): + return [ replace_tags(v) for v in value ] -_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) -def _remove_root(value): - m = _remove_root_re.search(value) - if m: - value = m.group(1) - return value +class RemoveRootAdaptor(AdaptorFunc): + _remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) + def _remove_root(self, value): + m = self._remove_root_re.search(value) + if m: + value = m.group(1) + return value -def remove_root(value): - return [ _remove_root(v) for v in value ] + def __call__(self, value): + return [ self._remove_root(v) for v in value ] -_clean_spaces_re = re.compile("\s+", re.U) -def remove_multispaces(value): - return [ _clean_spaces_re.sub(' ', v) for v in value ] +class CleanSpacesAdaptor(AdaptorFunc): + _clean_spaces_re = re.compile("\s+", re.U) + def __call__(self, value): + return [ self._clean_spaces_re.sub(' ', v) for v in value ] -def strip(value): - return [ v.strip() for v in value ] +class StripAdaptor(AdaptorFunc): + def __call__(self, value): + return [ v.strip() for v in value ] -def drop_empty_elements(value): - return [ v for v in value if v ] +class DropEmptyAdaptor(AdaptorFunc): + def __call__(self, value): + return [ v for v in value if v ] -def delist(value): - return ' '.join(value) +class DelistAdaptor(AdaptorFunc): + def __init__(self, kwargs={}): + self.delimiter = kwargs.get('join_delimiter', ' ') + + def __call__(self, value): + return self.delimiter.join(value) +class PickleAdaptor(AdaptorFunc): + def __call__(self, value): + return pickle.dumps(value) +class DePickleAdaptor(AdaptorFunc): + def __call__(self, value): + return pickle.loads(value) + +class UniqueAdaptor(AdaptorFunc): + def __call__(self, value): + return unique(value) + +class FlattenAdaptor(AdaptorFunc): + def __call__(self, value): + return flatten(value) ############# # Pipelines # @@ -200,42 +271,38 @@ def delist(value): The following methods automatically generate adaptor pipelines for some basic datatypes, according to the parameters you pass them. """ -def single_pipeline(remove_root=True, remove_tags=True, do_unquote=True): - pipe = [ extract, - unique, - to_unicode, - drop_empty_elements, - remove_multispaces, - strip, +def single_pipeline(do_remove_root=True, do_remove_tags=True, do_unquote=True): + pipe = [ ExtractAdaptor, + UniqueAdaptor, + ToUnicodeAdaptor, + DropEmptyAdaptor, + CleanSpacesAdaptor, + StripAdaptor, ] - if remove_root: - pipe.insert(4, remove_root) - if remove_tags: - pipe.insert(5, remove_tags) + + if do_remove_root: + pipe.insert(4, RemoveRootAdaptor) + if do_remove_tags: + pipe.insert(5, RemoveTagsAdaptor) if do_unquote: - pipe.append(unquote) - return pipe + [delist] + pipe.append(UnquoteAdaptor) + return pipe + [DelistAdaptor] def url_pipeline(): - return [ extract_links, - unique, - to_unicode, - drop_empty_elements, + return [ ExtractImagesAdaptor, + UniqueAdaptor, + ToUnicodeAdaptor, + DropEmptyAdaptor, ] -def list_pipeline(do_extract=True): - pipe = [] - if do_extract: - pipe.append(extract) - return pipe + [ unique, - to_unicode, - drop_empty_elements, - unquote, - remove_tags, - remove_root, - strip, - ] +list_pipeline = [ ExtractAdaptor, + UniqueAdaptor, + ToUnicodeAdaptor, + DropEmptyAdaptor, + UnquoteAdaptor, + RemoveTagsAdaptor, + RemoveRootAdaptor, + StripAdaptor, + ] -def list_join_pipeline(delimiter='\t'): - return list_pipeline() + [delimiter.join] - +list_join_pipeline = list_pipeline + [DelistAdaptor] diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index a68568104..7157dc761 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,4 +1,5 @@ from scrapy.item.adaptors import AdaptorPipe +from scrapy.conf import settings class ScrapedItem(object): """ @@ -7,18 +8,31 @@ class ScrapedItem(object): that identifies uniquely the given scraped item. """ - def setadaptors(self, adaptors_pipe): + def __init__(self, adaptors_pipe={}): + #self.set_adaptors(adaptors_pipe) + pass + + def set_adaptors(self, adaptors_pipe): """ - Set adaptors to use for this item. Receives a dict of adaptors and - returns the item itself. + Set the adaptors to use for this item. Receives a dict of the adaptors + desired for each attribute and returns the item itself. """ - object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe)) + setattr(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe)) return self - def append_adaptor(self, attrname, adaptor): - self._adaptors_pipe.append_adaptor(attrname, adaptor) - - def attribute(self, attrname, value): - value = self._adaptors_pipe.execute(attrname, value) - if not hasattr(self, attrname) or not getattr(self, attrname): - setattr(self, attrname, value) + def set_attrib_adaptors(self, attrib, adaptors): + """ + Set the adaptors (from a list or tuple) to be used for a specific attribute. + """ + self._adaptors_pipe.set_adaptors(attrib, adaptors) + + def attribute(self, attrname, value, **kwargs): + val = self._adaptors_pipe.execute(attrname, value, kwargs) + if not getattr(self, attrname, None) or kwargs.get('override'): + setattr(self, attrname, val) + + def __sub__(self, other): + raise NotImplementedError + +class ItemDelta(object): + pass diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py index df1627b5a..c8e1db286 100644 --- a/scrapy/trunk/scrapy/tests/test_adaptors.py +++ b/scrapy/trunk/scrapy/tests/test_adaptors.py @@ -7,17 +7,17 @@ from scrapy.http import Response, ResponseBody class AdaptorsTestCase(unittest.TestCase): def test_extract(self): sample_xsel = XmlXPathSelector(text='foobartest') - self.assertEqual(adaptors.extract(sample_xsel.x('/')), + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('/')), ['foobartest']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')), + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/*')), ['foobar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/@id')), ['2']) + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1')), ['foobar']) + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1//text()')), ['foo', 'bar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//text()')), ['foo', 'bar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag']) + self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag3/@value')), ['mytag']) def test_extract_links(self): test_data = """ @@ -32,53 +32,55 @@ class AdaptorsTestCase(unittest.TestCase): """ sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data)) sample_xsel = XmlXPathSelector(sample_response) - self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')), + sample_adaptor = adaptors.ExtractImagesAdaptor({'response': sample_response}) + + self.assertEqual(sample_adaptor(sample_xsel.x('//@href')), [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html']) - self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')), + self.assertEqual(sample_adaptor(sample_xsel.x('//a')), [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg']) - self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')), + self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')), [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html']) def test_to_unicode(self): - self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']), + self.assertEqual(adaptors.ToUnicodeAdaptor()(['lala', 'lele', 'luluñ', 1, 'áé']), [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9']) def test_regex(self): - func = adaptors.regex(r'href="(.*?)"') - self.assertEqual(func(['dsa', - 'href="lelelel.net"']), - ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) + adaptor = adaptors.RegexAdaptor({'regex': r'href="(.*?)"'}) + self.assertEqual(adaptor(['dsa', + 'href="lelelel.net"']), + ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) def test_unquote_all(self): - self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) + self.assertEqual(adaptors.UnquoteAdaptor({'keep': []})([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) def test_unquote(self): - self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) + self.assertEqual(adaptors.UnquoteAdaptor()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) def test_remove_tags(self): test_data = ['adsaas
', '
dsadasf
'] - self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf']) + self.assertEqual(adaptors.RemoveTagsAdaptor()(test_data), ['adsaas', 'dsadasf']) def test_remove_root(self): - self.assertEqual(adaptors.remove_root(['
lallaadsfsdfdspepepep
']), + self.assertEqual(adaptors.RemoveRootAdaptor()(['
lallaadsfsdfdspepepep
']), ['lallaadsfsdfdspepepep
']) def test_remove_multispaces(self): - self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']), + self.assertEqual(adaptors.CleanSpacesAdaptor()([' hello, whats up?', 'testing testingtesting testing']), [' hello, whats up?', 'testing testingtesting testing']) def test_strip(self): - self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), + self.assertEqual(adaptors.StripAdaptor()([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), ['hi there, sweety ;D', 'I CAN HAZ TEST??']) def test_drop_empty_elements(self): - self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']), + self.assertEqual(adaptors.DropEmptyAdaptor()([1, 2, None, 5, None, 6, None, 'hi']), [1, 2, 5, 6, 'hi']) def test_delist(self): - self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), + self.assertEqual(adaptors.DelistAdaptor()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), 'hi there fellas. this is my test.')