RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- Removed AdaptorFunc objects 

- Changed "AdaptorPipe" to "AdaptorDict"
- Moved adaptors to contrib/adaptors
- Fixed some tests

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40347
This commit is contained in:
elpolilla 2008-10-27 11:58:56 +00:00
parent ed98a84235
commit 4288cb3f17
8 changed files with 247 additions and 328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/trunk/scrapy/contrib/adaptors/__init__.py Normal file
View File

@ -0,0 +1,4 @@
from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, Delist, Regex
from scrapy.utils.python import unique, flatten

95
scrapy/trunk/scrapy/contrib/adaptors/extraction.py Normal file
View File

@ -0,0 +1,95 @@
"""
Adaptors related with extraction of data
"""
import urlparse
from scrapy.utils.python import flatten
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
def _extract(locations, extractor='extract'):
if isinstance(locations, (XPathSelector, XPathSelectorList)):
return flatten(getattr(locations, extractor)())
elif hasattr(locations, '__iter__'):
return flatten([getattr(x, extractor)() if isinstance(x, (XPathSelector, XPathSelectorList)) else x for x in flatten(locations)])
elif isinstance(locations, basestring):
return [locations]
else:
return []
def extract(locations):
"""
This adaptor extracts a list of strings
from 'locations', which can be either an iterable,
or an XPathSelector/XPathSelectorList.
Input: XPathSelector, XPathSelectorList, iterable, basestring
Output: list of unicodes
"""
return _extract(locations)
def extract_unquoted(locations):
"""
This adaptor extracts a list of unquoted strings
from 'locations', which can be either an iterable,
or an XPathSelector/XPathSelectorList.
The difference between this and the extract adaptor is
that this adaptor will only extract text nodes and unquote them.
Input: XPathSelector, XPathSelectorList, iterable, basestring
Output: list of unicodes
"""
return _extract(locations, 'extract_unquoted')
class ExtractImages(object):
"""
This adaptor receives either an XPathSelector containing
the desired locations for finding urls, or a list of relative
links to be resolved.
Input: XPathSelector, XPathSelectorList, iterable
Output: list of unicodes
"""
def __init__(self, response=None, base_url=None):
self.response = response
self.base_url = base_url
def extract_from_xpath(self, selector):
ret = []
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'a':
children = selector.x('child::*')
if len(children) > 1:
ret.extend(selector.x('.//@href'))
ret.extend(selector.x('.//@src'))
elif len(children) == 1 and children[0].xmlNode.name == 'img':
ret.extend(children.x('@src'))
else:
ret.extend(selector.x('@href'))
elif selector.xmlNode.name == 'img':
ret.extend(selector.x('@src'))
else:
ret.extend(selector.x('.//@href'))
ret.extend(selector.x('.//@src'))
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
ret.append(selector)
return ret
def __call__(self, locations):
if isinstance(locations, tuple) and len(locations) > 1:
self.base_url = locations[1]
locations = locations[0]
if not self.response and not self.base_url:
raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.')
rel_links = []
for location in flatten(locations):
if isinstance(location, (XPathSelector, XPathSelectorList)):
rel_links.extend(self.extract_from_xpath(location))
else:
rel_links.append(location)
rel_links = extract(rel_links)
base_url = self.base_url or self.response.url
return [urlparse.urljoin(base_url, link) for link in rel_links]

39
scrapy/trunk/scrapy/contrib/adaptors/markup.py Normal file
View File

@ -0,0 +1,39 @@
import re
from scrapy.utils.markup import replace_tags, remove_entities
def remove_tags(value):
"""
Input: iterable with strings
Output: list of strings
"""
return [ replace_tags(v) for v in value ]
def remove_root(value):
"""
Input: iterable with strings
Output: list of strings
"""
def _remove_root(value):
_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
m = _remove_root_re.search(value)
if m:
value = m.group(1)
return value
return [ _remove_root(v) for v in value ]
class Unquote(object):
"""
Receives a list of strings, removes all of the
entities the strings may have, and returns
a new list
Input: iterable with strings
Output: list of strings
"""
def __init__(self, keep=['lt', 'amp']):
self.keep = keep
def __call__(self, value):
return [ remove_entities(v, keep=self.keep) for v in value ]

59
scrapy/trunk/scrapy/contrib/adaptors/misc.py Normal file
View File

@ -0,0 +1,59 @@
import re
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
from scrapy.utils.misc import extract_regex
from scrapy.utils.python import flatten
def to_unicode(value):
"""
Receives a list of strings, converts
it to unicode, and returns a new list.
Input: iterable with strings
Output: list of unicodes
"""
if hasattr(value, '__iter__'):
return [ unicode(v) for v in value ]
else:
raise TypeError('to_unicode must receive an iterable.')
def clean_spaces(value):
_clean_spaces_re = re.compile("\s+", re.U)
return [ _clean_spaces_re.sub(' ', v) for v in value ]
def strip_list(value):
return [ v.strip() for v in value ]
def drop_empty(value):
return [ v for v in value if v ]
class Delist(object):
"""
Input: iterable with strings
Output: unicode
"""
def __init__(self, delimiter=' '):
self.delimiter = delimiter
def __call__(self, value):
return self.delimiter.join(value)
class Regex(object):
"""
This adaptor must receive either a list of strings or an XPathSelector
and return a new list with the matches of the given strings with the given regular
expression (which is passed by a keyword argument, and is mandatory for this adaptor).
Input: XPathSelector, XPathSelectorList, iterable
Output: list of unicodes
"""
def __init__(self, regex=r''):
self.regex = regex
def __call__(self, value):
if self.regex:
if isinstance(value, (XPathSelector, XPathSelectorList)):
return value.re(self.regex)
elif hasattr(value, '__iter__'):
return flatten([extract_regex(self.regex, string, 'utf-8') for string in value])
return value

7
scrapy/trunk/scrapy/contrib/item/models.py
View File

@ -37,12 +37,11 @@ class RobustScrapedItem(ScrapedItem):
'url': basestring, # the main URL where this item was scraped from
}
def __init__(self, data=None, adaptors_pipe={}):
def __init__(self, data=None):
"""
A scraped item can be initialised with a dictionary that will be
squirted directly into the object.
"""
super(RobustScrapedItem, self).__init__(adaptors_pipe)
if isinstance(data, dict):
for attr, value in data.iteritems():
setattr(self, attr, value)
@ -71,8 +70,8 @@ class RobustScrapedItem(ScrapedItem):
self.__dict__.pop(attr, None)
return
if attr == '_adaptors_pipe':
return object.__setattr__(self, '_adaptors_pipe', value)
if attr == '_adaptors_dict':
return object.__setattr__(self, '_adaptors_dict', value)
type1 = self.ATTRIBUTES[attr]
if hasattr(type1, '__iter__'):

295
scrapy/trunk/scrapy/item/adaptors.py
View File

@ -1,17 +1,7 @@
import re
import cPickle as pickle
from traceback import format_exc
from urlparse import urlparse
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
from scrapy.utils.python import unique, flatten
from scrapy.utils.markup import replace_tags, remove_entities
from scrapy.utils.misc import extract_regex
from scrapy.conf import settings
from scrapy import log
class AdaptorPipe(object):
class AdaptorDict(dict):
"""
Class that represents an item's attribute pipeline.
@ -19,290 +9,25 @@ class AdaptorPipe(object):
of adaptors to be run for filtering the input before storing.
"""
def __init__(self, adaptors_pipe=None):
"""
Receives a dictionary that maps attribute_name to a list of adaptor functions
"""
self.pipes = adaptors_pipe or {}
def set_adaptors(self, attr, adaptors):
"""
Set the adaptor pipeline that will be used for the specified attribute
"""
self.pipes[attr] = adaptors
def execute(self, attrname, value, kwargs):
def execute(self, attrname, value, debug=False):
"""
Execute pipeline for attribute name "attrname" and value "value".
"""
debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
debug = debug or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
for adaptor in self.pipes.get(attrname, []):
for adaptor in self.get(attrname, []):
name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__
try:
if debug:
print " %07s | input >" % adaptor.__name__, repr(value)
value = adaptor(kwargs)(value)
print " %07s | input >" % name, repr(value)
value = adaptor(value)
if debug:
print " %07s | output >" % adaptor.__name__, repr(value)
print " %07s | output >" % name, repr(value)
except Exception, e:
print "Error in '%s' adaptor. Traceback text:" % adaptor.__name__
except Exception:
print "Error in '%s' adaptor. Traceback text:" % name
print format_exc()
return
return value
class AdaptorFunc(object):
"""
This is the base class for adaptors.
An adaptor is just an object subclassed from this class
which defines the __call__ method, and receives/returns only
one value.
You can send the adaptor some extra options while creating it (just before running it)
through **kwargs, and managing them by overriding the __init__ method, as shown on
the UnquoteAdaptor, for example.
"""
def __init__(self, kwargs={}):
pass
def __call__(self):
raise NotImplementedError('You must define the __call__ method to create and use an adaptor')
############
# Adaptors #
############
class ExtractAdaptor(AdaptorFunc):
"""
This adaptor extracts a list of strings
from 'location', which can be either a list (or tuple),
or an XPathSelector.
This adaptor *always* returns a list.
"""
def __call__(self, location):
if not location:
return []
elif isinstance(location, (XPathSelector, XPathSelectorList)):
return flatten(location.extract())
elif isinstance(location, (list, tuple)):
return flatten(map(lambda x: x.extract() if isinstance(x, (XPathSelector, XPathSelectorList)) else x, flatten(location)))
elif isinstance(location, basestring):
return [location]
class ExtractImagesAdaptor(AdaptorFunc):
"""
This adaptor receives either an XPathSelector containing
the desired locations for finding urls, or a tuple like (xpath, regexp)
containing the xpath locations to look in, and a regular expression
to parse those locations.
In any case, this adaptor returns a list containing the absolute urls extracted.
"""
def __init__(self, kwargs):
self.base_url = kwargs.get('base_url')
self.response = kwargs.get('response')
if not self.response and not self.base_url:
raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.')
def extract_from_xpath(self, selector):
ret = []
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'a':
children = selector.x('child::*')
if len(children) > 1:
ret.extend(selector.x('.//@href'))
ret.extend(selector.x('.//@src'))
elif len(children) == 1 and children[0].xmlNode.name == 'img':
ret.extend(children.x('@src'))
else:
ret.extend(selector.x('@href'))
elif selector.xmlNode.name == 'img':
ret.extend(selector.x('@src'))
else:
ret.extend(selector.x('.//@href'))
ret.extend(selector.x('.//@src'))
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
ret.append(selector)
return ret
def absolutize_link(self, base_url, link):
base_url = urlparse(base_url)
ret = []
if link.startswith('/'):
ret.append('http://%s%s' % (base_url.hostname, link))
elif link.startswith('http://'):
ret.append(link)
else:
ret.append('http://%s%s/%s' % (base_url.hostname, base_url.path, link))
return ret
def __call__(self, locations):
rel_links = []
for location in flatten(locations):
if isinstance(location, (XPathSelector, XPathSelectorList)):
rel_links.extend(self.extract_from_xpath(location))
else:
rel_links.append(location)
rel_links = ExtractAdaptor()(rel_links)
if self.response:
return flatten([self.absolutize_link(self.response.url, link) for link in rel_links])
elif self.base_url:
return flatten([self.absolutize_link(self.base_url, link) for link in rel_links])
else:
abs_links = []
for link in rel_links:
if link.startswith('http://'):
abs_links.append(link)
else:
log.msg('Couldnt get the absolute url for "%s". Ignoring link...' % link, 'WARNING')
return abs_links
class BoolAdaptor(AdaptorFunc):
def __call__(self, value):
return bool(value)
class ToUnicodeAdaptor(AdaptorFunc):
"""
Receives a list of strings, converts
it to unicode, and returns a new list.
"""
def __call__(self, value):
if isinstance(value, (list, tuple)):
return [ unicode(v) for v in value ]
else:
raise TypeError('ToUnicodeAdaptor must receive either a list or a tuple.')
class RegexAdaptor(AdaptorFunc):
"""
This adaptor must receive either a list of strings or an XPathSelector
and return a new list with the matches of the given strings with the given regular
expression (which is passed by a keyword argument, and is mandatory for this adaptor).
"""
def __init__(self, kwargs):
self.regex = kwargs.get('regex')
def __call__(self, value):
if self.regex:
if isinstance(value, (XPathSelector, XPathSelectorList)):
return value.re(self.regex)
elif isinstance(value, list) and value:
return flatten([extract_regex(self.regex, string, 'utf-8') for string in value])
return value
class UnquoteAdaptor(AdaptorFunc):
"""
Receives a list of strings, removes all of the
entities the strings may have, and returns
a new list
"""
def __init__(self, kwargs={}):
self.keep = kwargs.get('keep', ['lt', 'amp'])
def __call__(self, value):
return [ remove_entities(v, keep=self.keep) for v in value ]
class RemoveTagsAdaptor(AdaptorFunc):
def __call__(self, value):
return [ replace_tags(v) for v in value ]
class RemoveRootAdaptor(AdaptorFunc):
_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
def _remove_root(self, value):
m = self._remove_root_re.search(value)
if m:
value = m.group(1)
return value
def __call__(self, value):
return [ self._remove_root(v) for v in value ]
class CleanSpacesAdaptor(AdaptorFunc):
_clean_spaces_re = re.compile("\s+", re.U)
def __call__(self, value):
return [ self._clean_spaces_re.sub(' ', v) for v in value ]
class StripAdaptor(AdaptorFunc):
def __call__(self, value):
return [ v.strip() for v in value ]
class DropEmptyAdaptor(AdaptorFunc):
def __call__(self, value):
return [ v for v in value if v ]
class DelistAdaptor(AdaptorFunc):
def __init__(self, kwargs={}):
self.delimiter = kwargs.get('join_delimiter', ' ')
def __call__(self, value):
return self.delimiter.join(value)
class PickleAdaptor(AdaptorFunc):
def __call__(self, value):
return pickle.dumps(value)
class DePickleAdaptor(AdaptorFunc):
def __call__(self, value):
return pickle.loads(value)
class UniqueAdaptor(AdaptorFunc):
def __call__(self, value):
return unique(value)
class FlattenAdaptor(AdaptorFunc):
def __call__(self, value):
return flatten(value)
#############
# Pipelines #
#############
"""
The following methods automatically generate adaptor pipelines
for some basic datatypes, according to the parameters you pass them.
"""
def single_pipeline(do_remove_root=True, do_remove_tags=True, do_unquote=True):
pipe = [ ExtractAdaptor,
UniqueAdaptor,
ToUnicodeAdaptor,
DropEmptyAdaptor,
CleanSpacesAdaptor,
StripAdaptor,
]
if do_remove_root:
pipe.insert(4, RemoveRootAdaptor)
if do_remove_tags:
pipe.insert(5, RemoveTagsAdaptor)
if do_unquote:
pipe.append(UnquoteAdaptor)
return pipe + [DelistAdaptor]
def url_pipeline():
return [ ExtractImagesAdaptor,
UniqueAdaptor,
ToUnicodeAdaptor,
DropEmptyAdaptor,
]
list_pipeline = [ ExtractAdaptor,
UniqueAdaptor,
ToUnicodeAdaptor,
DropEmptyAdaptor,
UnquoteAdaptor,
RemoveTagsAdaptor,
RemoveRootAdaptor,
StripAdaptor,
]
list_join_pipeline = list_pipeline + [DelistAdaptor]

16
scrapy/trunk/scrapy/item/models.py
View File

@ -1,4 +1,4 @@
from scrapy.item.adaptors import AdaptorPipe
from scrapy.item.adaptors import AdaptorDict
from scrapy.conf import settings
class ScrapedItem(object):
@ -8,23 +8,23 @@ class ScrapedItem(object):
that identifies uniquely the given scraped item.
"""
def set_adaptors(self, adaptors_pipe):
def set_adaptors(self, adaptors_dict, **kwargs):
"""
Set the adaptors to use for this item. Receives a dict of the adaptors
desired for each attribute and returns the item itself.
"""
setattr(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict))
return self
def set_attrib_adaptors(self, attrib, adaptors):
def set_attrib_adaptors(self, attrib, adaptors, **kwargs):
"""
Set the adaptors (from a list or tuple) to be used for a specific attribute.
"""
self._adaptors_pipe.set_adaptors(attrib, adaptors)
self._adaptors_dict[attrib] = adaptors
def attribute(self, attrname, value, **kwargs):
val = self._adaptors_pipe.execute(attrname, value, kwargs)
if not getattr(self, attrname, None) or kwargs.get('override'):
def attribute(self, attrname, value, debug=False, override=False):
val = self._adaptors_dict.execute(attrname, value, debug)
if not getattr(self, attrname, None) or override:
setattr(self, attrname, val)
def __sub__(self, other):

60
scrapy/trunk/scrapy/tests/test_adaptors.py
View File

@ -1,86 +1,84 @@
# -*- coding: utf8 -*-
import unittest
from scrapy.item import adaptors
from scrapy.xpath.selector import XmlXPathSelector
from scrapy.contrib import adaptors
from scrapy.http import Response, ResponseBody
from scrapy.xpath.selector import XmlXPathSelector
class AdaptorsTestCase(unittest.TestCase):
def test_extract(self):
sample_xsel = XmlXPathSelector(text='<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>')
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('/')),
self.assertEqual(adaptors.extract(sample_xsel.x('/')),
['<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/*')),
self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')),
['<tag1>foo<tag2>bar</tag2></tag1>', '<tag3 value="mytag">test</tag3>'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/@id')), ['2'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1//text()')),
['foo', 'bar'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//text()')),
['foo', 'bar', 'test'])
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag3/@value')), ['mytag'])
self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2'])
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), ['foo', 'bar'])
self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), ['foo', 'bar', 'test'])
self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag'])
def test_extract_links(self):
test_data = """<html><body>
<div>
<a href="lala1.html">lala1</a>
<a href="lala1/lala1.html">lala1</a>
<a href="/lala2.html">lala2</a>
<a href="http://foobar.com/lala3.html">lala3</a>
<a href="lala4.html"><img src="lala4.jpg" /></a>
<a href="http://foobar.com/pepepe/papapa/lala3.html">lala3</a>
<a href="lala4.html"><img src="/imgs/lala4.jpg" /></a>
<a onclick="javascript: opensomething('/my_html1.html');">something1</a>
<a onclick="javascript: opensomething('my_html2.html');">something2</a>
<a onclick="javascript: opensomething('dummy/my_html2.html');">something2</a>
</div>
</body></html>"""
sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data))
sample_xsel = XmlXPathSelector(sample_response)
sample_adaptor = adaptors.ExtractImagesAdaptor({'response': sample_response})
sample_adaptor = adaptors.ExtractImages(response=sample_response)
self.assertEqual(sample_adaptor(sample_xsel.x('//@href')),
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html'])
[u'http://foobar.com/lala1/lala1.html', u'http://foobar.com/lala2.html',
u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/lala4.html'])
self.assertEqual(sample_adaptor(sample_xsel.x('//a')),
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg'])
[u'http://foobar.com/lala1/lala1.html', u'http://foobar.com/lala2.html',
u'http://foobar.com/pepepe/papapa/lala3.html', u'http://foobar.com/imgs/lala4.jpg'])
self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')),
[u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
def test_to_unicode(self):
self.assertEqual(adaptors.ToUnicodeAdaptor()(['lala', 'lele', 'luluñ', 1, 'áé']),
self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
[u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
def test_regex(self):
adaptor = adaptors.RegexAdaptor({'regex': r'href="(.*?)"'})
adaptor = adaptors.Regex(regex=r'href="(.*?)"')
self.assertEqual(adaptor(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
'<a href="das.biz">href="lelelel.net"</a>']),
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
def test_unquote_all(self):
self.assertEqual(adaptors.UnquoteAdaptor({'keep': []})([u'hello&copy;&amp;welcome', u'&lt;br /&gt;&amp;']), [u'hello\xa9&welcome', u'<br />&'])
self.assertEqual(adaptors.Unquote(keep=[])([u'hello&copy;&amp;welcome', u'&lt;br /&gt;&amp;']), [u'hello\xa9&welcome', u'<br />&'])
def test_unquote(self):
self.assertEqual(adaptors.UnquoteAdaptor()([u'hello&copy;&amp;welcome', u'&lt;br /&gt;&amp;']), [u'hello\xa9&amp;welcome', u'&lt;br />&amp;'])
self.assertEqual(adaptors.Unquote()([u'hello&copy;&amp;welcome', u'&lt;br /&gt;&amp;']), [u'hello\xa9&amp;welcome', u'&lt;br />&amp;'])
def test_remove_tags(self):
test_data = ['<a href="lala">adsaas<br /></a>', '<div id="1"><table>dsadasf</table></div>']
self.assertEqual(adaptors.RemoveTagsAdaptor()(test_data), ['adsaas', 'dsadasf'])
self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
def test_remove_root(self):
self.assertEqual(adaptors.RemoveRootAdaptor()(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
self.assertEqual(adaptors.remove_root(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
['lallaa<a href="coso">dsfsdfds</a>pepepep<br />'])
def test_remove_multispaces(self):
self.assertEqual(adaptors.CleanSpacesAdaptor()([' hello, whats up?', 'testing testingtesting testing']),
self.assertEqual(adaptors.clean_spaces([' hello, whats up?', 'testing testingtesting testing']),
[' hello, whats up?', 'testing testingtesting testing'])
def test_strip(self):
self.assertEqual(adaptors.StripAdaptor()([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
def test_strip_list(self):
self.assertEqual(adaptors.strip_list([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
['hi there, sweety ;D', 'I CAN HAZ TEST??'])
def test_drop_empty_elements(self):
self.assertEqual(adaptors.DropEmptyAdaptor()([1, 2, None, 5, None, 6, None, 'hi']),
self.assertEqual(adaptors.drop_empty([1, 2, None, 5, None, 6, None, 'hi']),
[1, 2, 5, 6, 'hi'])
def test_delist(self):
self.assertEqual(adaptors.DelistAdaptor()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
'hi there fellas. this is my test.')