diff --git a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py new file mode 100644 index 000000000..97e76e48b --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py @@ -0,0 +1,4 @@ +from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages +from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote +from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, Delist, Regex +from scrapy.utils.python import unique, flatten diff --git a/scrapy/trunk/scrapy/contrib/adaptors/extraction.py b/scrapy/trunk/scrapy/contrib/adaptors/extraction.py new file mode 100644 index 000000000..c059bc350 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/adaptors/extraction.py @@ -0,0 +1,95 @@ +""" +Adaptors related with extraction of data +""" + +import urlparse +from scrapy.utils.python import flatten +from scrapy.xpath.selector import XPathSelector, XPathSelectorList + +def _extract(locations, extractor='extract'): + if isinstance(locations, (XPathSelector, XPathSelectorList)): + return flatten(getattr(locations, extractor)()) + elif hasattr(locations, '__iter__'): + return flatten([getattr(x, extractor)() if isinstance(x, (XPathSelector, XPathSelectorList)) else x for x in flatten(locations)]) + elif isinstance(locations, basestring): + return [locations] + else: + return [] + +def extract(locations): + """ + This adaptor extracts a list of strings + from 'locations', which can be either an iterable, + or an XPathSelector/XPathSelectorList. + + Input: XPathSelector, XPathSelectorList, iterable, basestring + Output: list of unicodes + """ + return _extract(locations) + +def extract_unquoted(locations): + """ + This adaptor extracts a list of unquoted strings + from 'locations', which can be either an iterable, + or an XPathSelector/XPathSelectorList. + The difference between this and the extract adaptor is + that this adaptor will only extract text nodes and unquote them. + + Input: XPathSelector, XPathSelectorList, iterable, basestring + Output: list of unicodes + """ + return _extract(locations, 'extract_unquoted') + +class ExtractImages(object): + """ + This adaptor receives either an XPathSelector containing + the desired locations for finding urls, or a list of relative + links to be resolved. + + Input: XPathSelector, XPathSelectorList, iterable + Output: list of unicodes + """ + def __init__(self, response=None, base_url=None): + self.response = response + self.base_url = base_url + + def extract_from_xpath(self, selector): + ret = [] + if selector.xmlNode.type == 'element': + if selector.xmlNode.name == 'a': + children = selector.x('child::*') + if len(children) > 1: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + elif len(children) == 1 and children[0].xmlNode.name == 'img': + ret.extend(children.x('@src')) + else: + ret.extend(selector.x('@href')) + elif selector.xmlNode.name == 'img': + ret.extend(selector.x('@src')) + else: + ret.extend(selector.x('.//@href')) + ret.extend(selector.x('.//@src')) + elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: + ret.append(selector) + + return ret + + def __call__(self, locations): + if isinstance(locations, tuple) and len(locations) > 1: + self.base_url = locations[1] + locations = locations[0] + + if not self.response and not self.base_url: + raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.') + + rel_links = [] + for location in flatten(locations): + if isinstance(location, (XPathSelector, XPathSelectorList)): + rel_links.extend(self.extract_from_xpath(location)) + else: + rel_links.append(location) + rel_links = extract(rel_links) + + base_url = self.base_url or self.response.url + return [urlparse.urljoin(base_url, link) for link in rel_links] diff --git a/scrapy/trunk/scrapy/contrib/adaptors/markup.py b/scrapy/trunk/scrapy/contrib/adaptors/markup.py new file mode 100644 index 000000000..b912f73e1 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/adaptors/markup.py @@ -0,0 +1,39 @@ +import re +from scrapy.utils.markup import replace_tags, remove_entities + +def remove_tags(value): + """ + Input: iterable with strings + Output: list of strings + """ + return [ replace_tags(v) for v in value ] + +def remove_root(value): + """ + Input: iterable with strings + Output: list of strings + """ + def _remove_root(value): + _remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) + m = _remove_root_re.search(value) + if m: + value = m.group(1) + return value + return [ _remove_root(v) for v in value ] + +class Unquote(object): + """ + Receives a list of strings, removes all of the + entities the strings may have, and returns + a new list + + Input: iterable with strings + Output: list of strings + """ + def __init__(self, keep=['lt', 'amp']): + self.keep = keep + + def __call__(self, value): + return [ remove_entities(v, keep=self.keep) for v in value ] + + diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py new file mode 100644 index 000000000..a93ad9f87 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -0,0 +1,59 @@ +import re +from scrapy.xpath.selector import XPathSelector, XPathSelectorList +from scrapy.utils.misc import extract_regex +from scrapy.utils.python import flatten + +def to_unicode(value): + """ + Receives a list of strings, converts + it to unicode, and returns a new list. + + Input: iterable with strings + Output: list of unicodes + """ + if hasattr(value, '__iter__'): + return [ unicode(v) for v in value ] + else: + raise TypeError('to_unicode must receive an iterable.') + +def clean_spaces(value): + _clean_spaces_re = re.compile("\s+", re.U) + return [ _clean_spaces_re.sub(' ', v) for v in value ] + +def strip_list(value): + return [ v.strip() for v in value ] + +def drop_empty(value): + return [ v for v in value if v ] + +class Delist(object): + """ + Input: iterable with strings + Output: unicode + """ + def __init__(self, delimiter=' '): + self.delimiter = delimiter + + def __call__(self, value): + return self.delimiter.join(value) + +class Regex(object): + """ + This adaptor must receive either a list of strings or an XPathSelector + and return a new list with the matches of the given strings with the given regular + expression (which is passed by a keyword argument, and is mandatory for this adaptor). + + Input: XPathSelector, XPathSelectorList, iterable + Output: list of unicodes + """ + def __init__(self, regex=r''): + self.regex = regex + + def __call__(self, value): + if self.regex: + if isinstance(value, (XPathSelector, XPathSelectorList)): + return value.re(self.regex) + elif hasattr(value, '__iter__'): + return flatten([extract_regex(self.regex, string, 'utf-8') for string in value]) + return value + diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py index 0f8679c47..2530d6814 100644 --- a/scrapy/trunk/scrapy/contrib/item/models.py +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -37,12 +37,11 @@ class RobustScrapedItem(ScrapedItem): 'url': basestring, # the main URL where this item was scraped from } - def __init__(self, data=None, adaptors_pipe={}): + def __init__(self, data=None): """ A scraped item can be initialised with a dictionary that will be squirted directly into the object. """ - super(RobustScrapedItem, self).__init__(adaptors_pipe) if isinstance(data, dict): for attr, value in data.iteritems(): setattr(self, attr, value) @@ -71,8 +70,8 @@ class RobustScrapedItem(ScrapedItem): self.__dict__.pop(attr, None) return - if attr == '_adaptors_pipe': - return object.__setattr__(self, '_adaptors_pipe', value) + if attr == '_adaptors_dict': + return object.__setattr__(self, '_adaptors_dict', value) type1 = self.ATTRIBUTES[attr] if hasattr(type1, '__iter__'): diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index 67e3243e4..2580c955b 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -1,17 +1,7 @@ -import re -import cPickle as pickle - from traceback import format_exc -from urlparse import urlparse - -from scrapy.xpath.selector import XPathSelector, XPathSelectorList -from scrapy.utils.python import unique, flatten -from scrapy.utils.markup import replace_tags, remove_entities -from scrapy.utils.misc import extract_regex from scrapy.conf import settings -from scrapy import log -class AdaptorPipe(object): +class AdaptorDict(dict): """ Class that represents an item's attribute pipeline. @@ -19,290 +9,25 @@ class AdaptorPipe(object): of adaptors to be run for filtering the input before storing. """ - def __init__(self, adaptors_pipe=None): - """ - Receives a dictionary that maps attribute_name to a list of adaptor functions - """ - self.pipes = adaptors_pipe or {} - - def set_adaptors(self, attr, adaptors): - """ - Set the adaptor pipeline that will be used for the specified attribute - """ - self.pipes[attr] = adaptors - - def execute(self, attrname, value, kwargs): + def execute(self, attrname, value, debug=False): """ Execute pipeline for attribute name "attrname" and value "value". """ - debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']) + debug = debug or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']) - for adaptor in self.pipes.get(attrname, []): + for adaptor in self.get(attrname, []): + name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__ try: if debug: - print " %07s | input >" % adaptor.__name__, repr(value) - value = adaptor(kwargs)(value) + print " %07s | input >" % name, repr(value) + value = adaptor(value) if debug: - print " %07s | output >" % adaptor.__name__, repr(value) + print " %07s | output >" % name, repr(value) - except Exception, e: - print "Error in '%s' adaptor. Traceback text:" % adaptor.__name__ + except Exception: + print "Error in '%s' adaptor. Traceback text:" % name print format_exc() return return value -class AdaptorFunc(object): - """ - This is the base class for adaptors. - - An adaptor is just an object subclassed from this class - which defines the __call__ method, and receives/returns only - one value. - - You can send the adaptor some extra options while creating it (just before running it) - through **kwargs, and managing them by overriding the __init__ method, as shown on - the UnquoteAdaptor, for example. - """ - def __init__(self, kwargs={}): - pass - - def __call__(self): - raise NotImplementedError('You must define the __call__ method to create and use an adaptor') - -############ -# Adaptors # -############ -class ExtractAdaptor(AdaptorFunc): - """ - This adaptor extracts a list of strings - from 'location', which can be either a list (or tuple), - or an XPathSelector. - - This adaptor *always* returns a list. - """ - - def __call__(self, location): - if not location: - return [] - elif isinstance(location, (XPathSelector, XPathSelectorList)): - return flatten(location.extract()) - elif isinstance(location, (list, tuple)): - return flatten(map(lambda x: x.extract() if isinstance(x, (XPathSelector, XPathSelectorList)) else x, flatten(location))) - elif isinstance(location, basestring): - return [location] - -class ExtractImagesAdaptor(AdaptorFunc): - """ - This adaptor receives either an XPathSelector containing - the desired locations for finding urls, or a tuple like (xpath, regexp) - containing the xpath locations to look in, and a regular expression - to parse those locations. - - In any case, this adaptor returns a list containing the absolute urls extracted. - """ - - def __init__(self, kwargs): - self.base_url = kwargs.get('base_url') - self.response = kwargs.get('response') - if not self.response and not self.base_url: - raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.') - - def extract_from_xpath(self, selector): - ret = [] - - if selector.xmlNode.type == 'element': - if selector.xmlNode.name == 'a': - children = selector.x('child::*') - if len(children) > 1: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif len(children) == 1 and children[0].xmlNode.name == 'img': - ret.extend(children.x('@src')) - else: - ret.extend(selector.x('@href')) - elif selector.xmlNode.name == 'img': - ret.extend(selector.x('@src')) - else: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: - ret.append(selector) - - return ret - - def absolutize_link(self, base_url, link): - base_url = urlparse(base_url) - ret = [] - - if link.startswith('/'): - ret.append('http://%s%s' % (base_url.hostname, link)) - elif link.startswith('http://'): - ret.append(link) - else: - ret.append('http://%s%s/%s' % (base_url.hostname, base_url.path, link)) - - return ret - - def __call__(self, locations): - rel_links = [] - for location in flatten(locations): - if isinstance(location, (XPathSelector, XPathSelectorList)): - rel_links.extend(self.extract_from_xpath(location)) - else: - rel_links.append(location) - rel_links = ExtractAdaptor()(rel_links) - - if self.response: - return flatten([self.absolutize_link(self.response.url, link) for link in rel_links]) - elif self.base_url: - return flatten([self.absolutize_link(self.base_url, link) for link in rel_links]) - else: - abs_links = [] - for link in rel_links: - if link.startswith('http://'): - abs_links.append(link) - else: - log.msg('Couldnt get the absolute url for "%s". Ignoring link...' % link, 'WARNING') - return abs_links - -class BoolAdaptor(AdaptorFunc): - def __call__(self, value): - return bool(value) - -class ToUnicodeAdaptor(AdaptorFunc): - """ - Receives a list of strings, converts - it to unicode, and returns a new list. - """ - - def __call__(self, value): - if isinstance(value, (list, tuple)): - return [ unicode(v) for v in value ] - else: - raise TypeError('ToUnicodeAdaptor must receive either a list or a tuple.') - -class RegexAdaptor(AdaptorFunc): - """ - This adaptor must receive either a list of strings or an XPathSelector - and return a new list with the matches of the given strings with the given regular - expression (which is passed by a keyword argument, and is mandatory for this adaptor). - """ - - def __init__(self, kwargs): - self.regex = kwargs.get('regex') - - def __call__(self, value): - if self.regex: - if isinstance(value, (XPathSelector, XPathSelectorList)): - return value.re(self.regex) - elif isinstance(value, list) and value: - return flatten([extract_regex(self.regex, string, 'utf-8') for string in value]) - return value - -class UnquoteAdaptor(AdaptorFunc): - """ - Receives a list of strings, removes all of the - entities the strings may have, and returns - a new list - """ - - def __init__(self, kwargs={}): - self.keep = kwargs.get('keep', ['lt', 'amp']) - - def __call__(self, value): - return [ remove_entities(v, keep=self.keep) for v in value ] - -class RemoveTagsAdaptor(AdaptorFunc): - def __call__(self, value): - return [ replace_tags(v) for v in value ] - -class RemoveRootAdaptor(AdaptorFunc): - _remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) - def _remove_root(self, value): - m = self._remove_root_re.search(value) - if m: - value = m.group(1) - return value - - def __call__(self, value): - return [ self._remove_root(v) for v in value ] - -class CleanSpacesAdaptor(AdaptorFunc): - _clean_spaces_re = re.compile("\s+", re.U) - def __call__(self, value): - return [ self._clean_spaces_re.sub(' ', v) for v in value ] - -class StripAdaptor(AdaptorFunc): - def __call__(self, value): - return [ v.strip() for v in value ] - -class DropEmptyAdaptor(AdaptorFunc): - def __call__(self, value): - return [ v for v in value if v ] - -class DelistAdaptor(AdaptorFunc): - def __init__(self, kwargs={}): - self.delimiter = kwargs.get('join_delimiter', ' ') - - def __call__(self, value): - return self.delimiter.join(value) - -class PickleAdaptor(AdaptorFunc): - def __call__(self, value): - return pickle.dumps(value) - -class DePickleAdaptor(AdaptorFunc): - def __call__(self, value): - return pickle.loads(value) - -class UniqueAdaptor(AdaptorFunc): - def __call__(self, value): - return unique(value) - -class FlattenAdaptor(AdaptorFunc): - def __call__(self, value): - return flatten(value) - -############# -# Pipelines # -############# -""" -The following methods automatically generate adaptor pipelines -for some basic datatypes, according to the parameters you pass them. -""" -def single_pipeline(do_remove_root=True, do_remove_tags=True, do_unquote=True): - pipe = [ ExtractAdaptor, - UniqueAdaptor, - ToUnicodeAdaptor, - DropEmptyAdaptor, - CleanSpacesAdaptor, - StripAdaptor, - ] - - if do_remove_root: - pipe.insert(4, RemoveRootAdaptor) - if do_remove_tags: - pipe.insert(5, RemoveTagsAdaptor) - if do_unquote: - pipe.append(UnquoteAdaptor) - return pipe + [DelistAdaptor] - -def url_pipeline(): - return [ ExtractImagesAdaptor, - UniqueAdaptor, - ToUnicodeAdaptor, - DropEmptyAdaptor, - ] - -list_pipeline = [ ExtractAdaptor, - UniqueAdaptor, - ToUnicodeAdaptor, - DropEmptyAdaptor, - UnquoteAdaptor, - RemoveTagsAdaptor, - RemoveRootAdaptor, - StripAdaptor, - ] - -list_join_pipeline = list_pipeline + [DelistAdaptor] diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 771d3a9b4..8a5bfe2c9 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,4 +1,4 @@ -from scrapy.item.adaptors import AdaptorPipe +from scrapy.item.adaptors import AdaptorDict from scrapy.conf import settings class ScrapedItem(object): @@ -8,23 +8,23 @@ class ScrapedItem(object): that identifies uniquely the given scraped item. """ - def set_adaptors(self, adaptors_pipe): + def set_adaptors(self, adaptors_dict, **kwargs): """ Set the adaptors to use for this item. Receives a dict of the adaptors desired for each attribute and returns the item itself. """ - setattr(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe)) + setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict)) return self - def set_attrib_adaptors(self, attrib, adaptors): + def set_attrib_adaptors(self, attrib, adaptors, **kwargs): """ Set the adaptors (from a list or tuple) to be used for a specific attribute. """ - self._adaptors_pipe.set_adaptors(attrib, adaptors) + self._adaptors_dict[attrib] = adaptors - def attribute(self, attrname, value, **kwargs): - val = self._adaptors_pipe.execute(attrname, value, kwargs) - if not getattr(self, attrname, None) or kwargs.get('override'): + def attribute(self, attrname, value, debug=False, override=False): + val = self._adaptors_dict.execute(attrname, value, debug) + if not getattr(self, attrname, None) or override: setattr(self, attrname, val) def __sub__(self, other): diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py index c8e1db286..63b4d3a1f 100644 --- a/scrapy/trunk/scrapy/tests/test_adaptors.py +++ b/scrapy/trunk/scrapy/tests/test_adaptors.py @@ -1,86 +1,84 @@ # -*- coding: utf8 -*- import unittest -from scrapy.item import adaptors -from scrapy.xpath.selector import XmlXPathSelector +from scrapy.contrib import adaptors from scrapy.http import Response, ResponseBody +from scrapy.xpath.selector import XmlXPathSelector class AdaptorsTestCase(unittest.TestCase): def test_extract(self): sample_xsel = XmlXPathSelector(text='foobartest') - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('/')), + self.assertEqual(adaptors.extract(sample_xsel.x('/')), ['foobartest']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/*')), + self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')), ['foobar', 'test']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/@id')), ['2']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1')), ['foobar']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1//text()')), - ['foo', 'bar']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//text()')), - ['foo', 'bar', 'test']) - self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag3/@value')), ['mytag']) + self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), ['foo', 'bar']) + self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), ['foo', 'bar', 'test']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag']) def test_extract_links(self): test_data = """
- lala1 + lala1 lala2 - lala3 - + lala3 + something1 - something2 + something2
""" sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data)) sample_xsel = XmlXPathSelector(sample_response) - sample_adaptor = adaptors.ExtractImagesAdaptor({'response': sample_response}) + sample_adaptor = adaptors.ExtractImages(response=sample_response) self.assertEqual(sample_adaptor(sample_xsel.x('//@href')), - [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', - u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html']) + [u'http://foobar.com/lala1/lala1.html', u'http://foobar.com/lala2.html', + u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/lala4.html']) self.assertEqual(sample_adaptor(sample_xsel.x('//a')), - [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', - u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg']) + [u'http://foobar.com/lala1/lala1.html', u'http://foobar.com/lala2.html', + u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/imgs/lala4.jpg']) self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')), [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html']) def test_to_unicode(self): - self.assertEqual(adaptors.ToUnicodeAdaptor()(['lala', 'lele', 'luluñ', 1, 'áé']), + self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']), [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9']) def test_regex(self): - adaptor = adaptors.RegexAdaptor({'regex': r'href="(.*?)"'}) + adaptor = adaptors.Regex(regex=r'href="(.*?)"') self.assertEqual(adaptor(['dsa', 'href="lelelel.net"']), ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) def test_unquote_all(self): - self.assertEqual(adaptors.UnquoteAdaptor({'keep': []})([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) + self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) def test_unquote(self): - self.assertEqual(adaptors.UnquoteAdaptor()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) + self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) def test_remove_tags(self): test_data = ['adsaas
', '
dsadasf
'] - self.assertEqual(adaptors.RemoveTagsAdaptor()(test_data), ['adsaas', 'dsadasf']) + self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf']) def test_remove_root(self): - self.assertEqual(adaptors.RemoveRootAdaptor()(['
lallaadsfsdfdspepepep
']), + self.assertEqual(adaptors.remove_root(['
lallaadsfsdfdspepepep
']), ['lallaadsfsdfdspepepep
']) def test_remove_multispaces(self): - self.assertEqual(adaptors.CleanSpacesAdaptor()([' hello, whats up?', 'testing testingtesting testing']), + self.assertEqual(adaptors.clean_spaces([' hello, whats up?', 'testing testingtesting testing']), [' hello, whats up?', 'testing testingtesting testing']) - def test_strip(self): - self.assertEqual(adaptors.StripAdaptor()([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), + def test_strip_list(self): + self.assertEqual(adaptors.strip_list([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), ['hi there, sweety ;D', 'I CAN HAZ TEST??']) def test_drop_empty_elements(self): - self.assertEqual(adaptors.DropEmptyAdaptor()([1, 2, None, 5, None, 6, None, 'hi']), + self.assertEqual(adaptors.drop_empty([1, 2, None, 5, None, 6, None, 'hi']), [1, 2, 5, 6, 'hi']) def test_delist(self): - self.assertEqual(adaptors.DelistAdaptor()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), + self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), 'hi there fellas. this is my test.')