mirror of https://github.com/scrapy/scrapy.git
Improved adaptors code and fixed some tests related with that
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40339
This commit is contained in:
elpolilla
parent
91ddfd6f80
commit
06c2509634
|
|
@ -1,197 +1,268 @@
|
|||
import re
|
||||
import cPickle as pickle
|
||||
|
||||
from traceback import format_exc
|
||||
from urlparse import urlparse
|
||||
|
||||
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
|
||||
from scrapy.utils.python import unique, flatten
|
||||
from scrapy.utils.markup import replace_tags, remove_entities
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log
|
||||
|
||||
class AdaptorPipe(object):
|
||||
"""
|
||||
Class that represents an item's attribute pipeline.
|
||||
|
||||
This class contains a dictionary of attributes, matched with a list
|
||||
of adaptors to be run for filtering the input before storing.
|
||||
"""
|
||||
|
||||
class AdaptorPipe:
|
||||
def __init__(self, adaptors_pipe=None):
|
||||
"""
|
||||
Receives a dictionary that maps attribute_name to a list of adaptor functions
|
||||
"""
|
||||
self.pipes = adaptors_pipe or {}
|
||||
|
||||
def append_adaptor(self, attrname, adaptor):
|
||||
def set_adaptors(self, attr, adaptors):
|
||||
"""
|
||||
Add an adaptor at the end of the provided attribute's pipeline
|
||||
Set the adaptor pipeline that will be used for the specified attribute
|
||||
"""
|
||||
if callable(adaptor):
|
||||
if self.pipes.get(attrname):
|
||||
self.pipes[attrname].append(adaptor)
|
||||
else:
|
||||
self.pipes[attrname] = [adaptor]
|
||||
|
||||
def execute(self, attrname, value, debug=False):
|
||||
self.pipes[attr] = adaptors
|
||||
|
||||
def execute(self, attrname, value, kwargs):
|
||||
"""
|
||||
Execute pipeline for attribute name "attrname" and value "value".
|
||||
"""
|
||||
for function in self.pipes.get(attrname, []):
|
||||
debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
|
||||
|
||||
for adaptor in self.pipes.get(attrname, []):
|
||||
try:
|
||||
if debug:
|
||||
print " %07s | input >" % function.func_name, repr(value)
|
||||
value = function(value)
|
||||
print " %07s | input >" % adaptor.__name__, repr(value)
|
||||
value = adaptor(kwargs)(value)
|
||||
if debug:
|
||||
print " %07s | output>" % function.func_name, repr(value)
|
||||
|
||||
print " %07s | output >" % adaptor.__name__, repr(value)
|
||||
|
||||
except Exception, e:
|
||||
print "Error in '%s' adaptor. Traceback text:" % function.func_name
|
||||
print "Error in '%s' adaptor. Traceback text:" % adaptor.__name__
|
||||
print format_exc()
|
||||
return
|
||||
|
||||
|
||||
return value
|
||||
|
||||
class AdaptorFunc(object):
|
||||
"""
|
||||
This is the base class for adaptors.
|
||||
|
||||
An adaptor is just an object subclassed from this class
|
||||
which defines the __call__ method, and receives/returns only
|
||||
one value.
|
||||
|
||||
You can send the adaptor some extra options while creating it (just before running it)
|
||||
through **kwargs, and managing them by overriding the __init__ method, as shown on
|
||||
the UnquoteAdaptor, for example.
|
||||
"""
|
||||
def __init__(self, kwargs={}):
|
||||
pass
|
||||
|
||||
def __call__(self):
|
||||
raise NotImplementedError('You must define the __call__ method to create and use an adaptor')
|
||||
|
||||
############
|
||||
# Adaptors #
|
||||
############
|
||||
def extract(location):
|
||||
class ExtractAdaptor(AdaptorFunc):
|
||||
"""
|
||||
This adaptor extracts a list of strings
|
||||
from 'location', which can be either a list (or tuple),
|
||||
or an XPathSelector.
|
||||
|
||||
This function *always* returns a list.
|
||||
This adaptor *always* returns a list.
|
||||
"""
|
||||
if not location:
|
||||
return []
|
||||
elif isinstance(location, XPathSelectorList):
|
||||
return flatten([extract(o) for o in location])
|
||||
elif isinstance(location, XPathSelector):
|
||||
return location.extract()
|
||||
elif isinstance(location, (list, tuple)):
|
||||
return flatten(location)
|
||||
elif isinstance(location, basestring):
|
||||
return [location]
|
||||
|
||||
def _absolutize_links(rel_links, current_url, base_url):
|
||||
abs_links = []
|
||||
for link in rel_links:
|
||||
if link.startswith('/'):
|
||||
abs_links.append('%s%s' % (base_url, link))
|
||||
elif link.startswith('http://'):
|
||||
abs_links.append(link)
|
||||
else:
|
||||
abs_links.append('%s/%s' % (current_url, link))
|
||||
return abs_links
|
||||
|
||||
def extract_links(locations):
|
||||
def __call__(self, location):
|
||||
if not location:
|
||||
return []
|
||||
elif isinstance(location, (XPathSelector, XPathSelectorList)):
|
||||
return flatten(location.extract())
|
||||
elif isinstance(location, (list, tuple)):
|
||||
return flatten(map(lambda x: x.extract() if isinstance(x, (XPathSelector, XPathSelectorList)) else x, flatten(location)))
|
||||
elif isinstance(location, basestring):
|
||||
return [location]
|
||||
|
||||
class ExtractImagesAdaptor(AdaptorFunc):
|
||||
"""
|
||||
This adaptor receives either an XPathSelector containing
|
||||
the desired locations for finding urls, or a tuple like (xpath, regexp)
|
||||
containing the xpath locations to look in, and a regular expression
|
||||
to parse those locations.
|
||||
|
||||
In any case, this adaptor returns a list of absolute urls extracted.
|
||||
In any case, this adaptor returns a list containing the absolute urls extracted.
|
||||
"""
|
||||
ret = []
|
||||
if locations:
|
||||
regexp = None
|
||||
if isinstance(locations, XPathSelector):
|
||||
locations = XPathSelectorList([locations])
|
||||
elif isinstance(locations, tuple):
|
||||
locations, regexp = locations
|
||||
|
||||
if isinstance(locations, XPathSelectorList):
|
||||
if regexp:
|
||||
ret = locations.re(regexp)
|
||||
else:
|
||||
for selector in locations:
|
||||
if selector.xmlNode.type == 'element':
|
||||
if selector.xmlNode.name == 'a':
|
||||
children = selector.x('child::*')
|
||||
if len(children) > 1:
|
||||
ret.extend(selector.x('.//@href'))
|
||||
ret.extend(selector.x('.//@src'))
|
||||
elif len(children) == 1 and children[0].xmlNode.name == 'img':
|
||||
ret.extend(children.x('@src'))
|
||||
else:
|
||||
ret.extend(selector.x('@href'))
|
||||
elif selector.xmlNode.name == 'img':
|
||||
ret.extend(selector.x('@src'))
|
||||
else:
|
||||
ret.extend(selector.x('.//@href'))
|
||||
ret.extend(selector.x('.//@src'))
|
||||
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
|
||||
ret.append(selector)
|
||||
ret = [selector.extract() for selector in ret]
|
||||
current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups()
|
||||
ret = _absolutize_links(ret, current_url, base_url)
|
||||
return ret
|
||||
|
||||
def to_unicode(value):
|
||||
def __init__(self, kwargs):
|
||||
self.base_url = kwargs.get('base_url')
|
||||
self.response = kwargs.get('response')
|
||||
if not self.response and not self.base_url:
|
||||
raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.')
|
||||
|
||||
def extract_from_xpath(self, selector):
|
||||
ret = []
|
||||
|
||||
if selector.xmlNode.type == 'element':
|
||||
if selector.xmlNode.name == 'a':
|
||||
children = selector.x('child::*')
|
||||
if len(children) > 1:
|
||||
ret.extend(selector.x('.//@href'))
|
||||
ret.extend(selector.x('.//@src'))
|
||||
elif len(children) == 1 and children[0].xmlNode.name == 'img':
|
||||
ret.extend(children.x('@src'))
|
||||
else:
|
||||
ret.extend(selector.x('@href'))
|
||||
elif selector.xmlNode.name == 'img':
|
||||
ret.extend(selector.x('@src'))
|
||||
else:
|
||||
ret.extend(selector.x('.//@href'))
|
||||
ret.extend(selector.x('.//@src'))
|
||||
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
|
||||
ret.append(selector)
|
||||
|
||||
return ret
|
||||
|
||||
def absolutize_link(self, base_url, link):
|
||||
base_url = urlparse(base_url)
|
||||
ret = []
|
||||
|
||||
if link.startswith('/'):
|
||||
ret.append('http://%s%s' % (base_url.hostname, link))
|
||||
elif link.startswith('http://'):
|
||||
ret.append(link)
|
||||
else:
|
||||
ret.append('http://%s%s/%s' % (base_url.hostname, base_url.path, link))
|
||||
|
||||
return ret
|
||||
|
||||
def __call__(self, locations):
|
||||
rel_links = []
|
||||
for location in flatten(locations):
|
||||
if isinstance(location, (XPathSelector, XPathSelectorList)):
|
||||
rel_links.extend(self.extract_from_xpath(location))
|
||||
else:
|
||||
rel_links.append(location)
|
||||
rel_links = ExtractAdaptor()(rel_links)
|
||||
|
||||
if self.response:
|
||||
return flatten([self.absolutize_link(self.response.url, link) for link in rel_links])
|
||||
elif self.base_url:
|
||||
return flatten([self.absolutize_link(self.base_url, link) for link in rel_links])
|
||||
else:
|
||||
abs_links = []
|
||||
for link in rel_links:
|
||||
if link.startswith('http://'):
|
||||
abs_links.append(link)
|
||||
else:
|
||||
log.msg('Couldnt get the absolute url for "%s". Ignoring link...' % link, 'WARNING')
|
||||
return abs_links
|
||||
|
||||
class BoolAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return bool(value)
|
||||
|
||||
class ToUnicodeAdaptor(AdaptorFunc):
|
||||
"""
|
||||
Receives a list of strings, converts
|
||||
it to unicode, and returns a new list.
|
||||
"""
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [ unicode(v) for v in value ]
|
||||
else:
|
||||
raise TypeError('to_unicode adaptor must receive a list or a tuple.')
|
||||
|
||||
def regex(expr):
|
||||
"""
|
||||
This factory function returns a ready-to-use
|
||||
adaptor for the specified regular expression.
|
||||
This adaptor will accept either an XPathSelectorList
|
||||
or a list of strings, and will apply the provided regular
|
||||
expression to each of its members.
|
||||
def __call__(self, value):
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [ unicode(v) for v in value ]
|
||||
else:
|
||||
raise TypeError('ToUnicodeAdaptor must receive either a list or a tuple.')
|
||||
|
||||
This adaptor always returns a list of strings.
|
||||
class RegexAdaptor(AdaptorFunc):
|
||||
"""
|
||||
def _regex(value):
|
||||
if isinstance(value, (XPathSelector, XPathSelectorList)):
|
||||
return value.re(expr)
|
||||
elif isinstance(value, list) and value:
|
||||
return flatten([extract_regex(expr, string, 'utf-8') for string in value])
|
||||
return value
|
||||
return _regex
|
||||
This adaptor must receive either a list of strings or an XPathSelector
|
||||
and return a new list with the matches of the given strings with the given regular
|
||||
expression (which is passed by a keyword argument, and is mandatory for this adaptor).
|
||||
"""
|
||||
|
||||
def __init__(self, kwargs):
|
||||
self.regex = kwargs.get('regex')
|
||||
|
||||
def unquote_all(value):
|
||||
def __call__(self, value):
|
||||
if self.regex:
|
||||
if isinstance(value, (XPathSelector, XPathSelectorList)):
|
||||
return value.re(self.regex)
|
||||
elif isinstance(value, list) and value:
|
||||
return flatten([extract_regex(self.regex, string, 'utf-8') for string in value])
|
||||
return value
|
||||
|
||||
class UnquoteAdaptor(AdaptorFunc):
|
||||
"""
|
||||
Receives a list of strings, removes all of the
|
||||
entities the strings may have, and returns
|
||||
a new list
|
||||
"""
|
||||
return [ remove_entities(v) for v in value ]
|
||||
|
||||
def unquote(value):
|
||||
"""
|
||||
Receives a list of strings, removes all of the entities
|
||||
the strings may have (except for < and &), and
|
||||
returns a new list
|
||||
"""
|
||||
return [ remove_entities(v, keep=['lt', 'amp']) for v in value ]
|
||||
def __init__(self, kwargs={}):
|
||||
self.keep = kwargs.get('keep', ['lt', 'amp'])
|
||||
|
||||
def remove_tags(value):
|
||||
return [ replace_tags(v) for v in value ]
|
||||
def __call__(self, value):
|
||||
return [ remove_entities(v, keep=self.keep) for v in value ]
|
||||
|
||||
class RemoveTagsAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return [ replace_tags(v) for v in value ]
|
||||
|
||||
_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
|
||||
def _remove_root(value):
|
||||
m = _remove_root_re.search(value)
|
||||
if m:
|
||||
value = m.group(1)
|
||||
return value
|
||||
class RemoveRootAdaptor(AdaptorFunc):
|
||||
_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
|
||||
def _remove_root(self, value):
|
||||
m = self._remove_root_re.search(value)
|
||||
if m:
|
||||
value = m.group(1)
|
||||
return value
|
||||
|
||||
def remove_root(value):
|
||||
return [ _remove_root(v) for v in value ]
|
||||
def __call__(self, value):
|
||||
return [ self._remove_root(v) for v in value ]
|
||||
|
||||
_clean_spaces_re = re.compile("\s+", re.U)
|
||||
def remove_multispaces(value):
|
||||
return [ _clean_spaces_re.sub(' ', v) for v in value ]
|
||||
class CleanSpacesAdaptor(AdaptorFunc):
|
||||
_clean_spaces_re = re.compile("\s+", re.U)
|
||||
def __call__(self, value):
|
||||
return [ self._clean_spaces_re.sub(' ', v) for v in value ]
|
||||
|
||||
def strip(value):
|
||||
return [ v.strip() for v in value ]
|
||||
class StripAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return [ v.strip() for v in value ]
|
||||
|
||||
def drop_empty_elements(value):
|
||||
return [ v for v in value if v ]
|
||||
class DropEmptyAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return [ v for v in value if v ]
|
||||
|
||||
def delist(value):
|
||||
return ' '.join(value)
|
||||
class DelistAdaptor(AdaptorFunc):
|
||||
def __init__(self, kwargs={}):
|
||||
self.delimiter = kwargs.get('join_delimiter', ' ')
|
||||
|
||||
def __call__(self, value):
|
||||
return self.delimiter.join(value)
|
||||
|
||||
class PickleAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return pickle.dumps(value)
|
||||
|
||||
class DePickleAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return pickle.loads(value)
|
||||
|
||||
class UniqueAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return unique(value)
|
||||
|
||||
class FlattenAdaptor(AdaptorFunc):
|
||||
def __call__(self, value):
|
||||
return flatten(value)
|
||||
|
||||
#############
|
||||
# Pipelines #
|
||||
|
|
@ -200,42 +271,38 @@ def delist(value):
|
|||
The following methods automatically generate adaptor pipelines
|
||||
for some basic datatypes, according to the parameters you pass them.
|
||||
"""
|
||||
def single_pipeline(remove_root=True, remove_tags=True, do_unquote=True):
|
||||
pipe = [ extract,
|
||||
unique,
|
||||
to_unicode,
|
||||
drop_empty_elements,
|
||||
remove_multispaces,
|
||||
strip,
|
||||
def single_pipeline(do_remove_root=True, do_remove_tags=True, do_unquote=True):
|
||||
pipe = [ ExtractAdaptor,
|
||||
UniqueAdaptor,
|
||||
ToUnicodeAdaptor,
|
||||
DropEmptyAdaptor,
|
||||
CleanSpacesAdaptor,
|
||||
StripAdaptor,
|
||||
]
|
||||
if remove_root:
|
||||
pipe.insert(4, remove_root)
|
||||
if remove_tags:
|
||||
pipe.insert(5, remove_tags)
|
||||
|
||||
if do_remove_root:
|
||||
pipe.insert(4, RemoveRootAdaptor)
|
||||
if do_remove_tags:
|
||||
pipe.insert(5, RemoveTagsAdaptor)
|
||||
if do_unquote:
|
||||
pipe.append(unquote)
|
||||
return pipe + [delist]
|
||||
pipe.append(UnquoteAdaptor)
|
||||
return pipe + [DelistAdaptor]
|
||||
|
||||
def url_pipeline():
|
||||
return [ extract_links,
|
||||
unique,
|
||||
to_unicode,
|
||||
drop_empty_elements,
|
||||
return [ ExtractImagesAdaptor,
|
||||
UniqueAdaptor,
|
||||
ToUnicodeAdaptor,
|
||||
DropEmptyAdaptor,
|
||||
]
|
||||
|
||||
def list_pipeline(do_extract=True):
|
||||
pipe = []
|
||||
if do_extract:
|
||||
pipe.append(extract)
|
||||
return pipe + [ unique,
|
||||
to_unicode,
|
||||
drop_empty_elements,
|
||||
unquote,
|
||||
remove_tags,
|
||||
remove_root,
|
||||
strip,
|
||||
]
|
||||
list_pipeline = [ ExtractAdaptor,
|
||||
UniqueAdaptor,
|
||||
ToUnicodeAdaptor,
|
||||
DropEmptyAdaptor,
|
||||
UnquoteAdaptor,
|
||||
RemoveTagsAdaptor,
|
||||
RemoveRootAdaptor,
|
||||
StripAdaptor,
|
||||
]
|
||||
|
||||
def list_join_pipeline(delimiter='\t'):
|
||||
return list_pipeline() + [delimiter.join]
|
||||
|
||||
list_join_pipeline = list_pipeline + [DelistAdaptor]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from scrapy.item.adaptors import AdaptorPipe
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ScrapedItem(object):
|
||||
"""
|
||||
|
|
@ -7,18 +8,31 @@ class ScrapedItem(object):
|
|||
that identifies uniquely the given scraped item.
|
||||
"""
|
||||
|
||||
def setadaptors(self, adaptors_pipe):
|
||||
def __init__(self, adaptors_pipe={}):
|
||||
#self.set_adaptors(adaptors_pipe)
|
||||
pass
|
||||
|
||||
def set_adaptors(self, adaptors_pipe):
|
||||
"""
|
||||
Set adaptors to use for this item. Receives a dict of adaptors and
|
||||
returns the item itself.
|
||||
Set the adaptors to use for this item. Receives a dict of the adaptors
|
||||
desired for each attribute and returns the item itself.
|
||||
"""
|
||||
object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
|
||||
setattr(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
|
||||
return self
|
||||
|
||||
def append_adaptor(self, attrname, adaptor):
|
||||
self._adaptors_pipe.append_adaptor(attrname, adaptor)
|
||||
|
||||
def attribute(self, attrname, value):
|
||||
value = self._adaptors_pipe.execute(attrname, value)
|
||||
if not hasattr(self, attrname) or not getattr(self, attrname):
|
||||
setattr(self, attrname, value)
|
||||
def set_attrib_adaptors(self, attrib, adaptors):
|
||||
"""
|
||||
Set the adaptors (from a list or tuple) to be used for a specific attribute.
|
||||
"""
|
||||
self._adaptors_pipe.set_adaptors(attrib, adaptors)
|
||||
|
||||
def attribute(self, attrname, value, **kwargs):
|
||||
val = self._adaptors_pipe.execute(attrname, value, kwargs)
|
||||
if not getattr(self, attrname, None) or kwargs.get('override'):
|
||||
setattr(self, attrname, val)
|
||||
|
||||
def __sub__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
class ItemDelta(object):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -7,17 +7,17 @@ from scrapy.http import Response, ResponseBody
|
|||
class AdaptorsTestCase(unittest.TestCase):
|
||||
def test_extract(self):
|
||||
sample_xsel = XmlXPathSelector(text='<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>')
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('/')),
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('/')),
|
||||
['<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')),
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/*')),
|
||||
['<tag1>foo<tag2>bar</tag2></tag1>', '<tag3 value="mytag">test</tag3>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')),
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('xml/@id')), ['2'])
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag1//text()')),
|
||||
['foo', 'bar'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//text()')),
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//text()')),
|
||||
['foo', 'bar', 'test'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag'])
|
||||
self.assertEqual(adaptors.ExtractAdaptor()(sample_xsel.x('//tag3/@value')), ['mytag'])
|
||||
|
||||
def test_extract_links(self):
|
||||
test_data = """<html><body>
|
||||
|
|
@ -32,53 +32,55 @@ class AdaptorsTestCase(unittest.TestCase):
|
|||
</body></html>"""
|
||||
sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data))
|
||||
sample_xsel = XmlXPathSelector(sample_response)
|
||||
self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')),
|
||||
sample_adaptor = adaptors.ExtractImagesAdaptor({'response': sample_response})
|
||||
|
||||
self.assertEqual(sample_adaptor(sample_xsel.x('//@href')),
|
||||
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
|
||||
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html'])
|
||||
self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')),
|
||||
self.assertEqual(sample_adaptor(sample_xsel.x('//a')),
|
||||
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
|
||||
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg'])
|
||||
self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')),
|
||||
self.assertEqual(sample_adaptor(sample_xsel.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')),
|
||||
[u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
|
||||
|
||||
def test_to_unicode(self):
|
||||
self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
|
||||
self.assertEqual(adaptors.ToUnicodeAdaptor()(['lala', 'lele', 'luluñ', 1, 'áé']),
|
||||
[u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
|
||||
|
||||
def test_regex(self):
|
||||
func = adaptors.regex(r'href="(.*?)"')
|
||||
self.assertEqual(func(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
|
||||
'<a href="das.biz">href="lelelel.net"</a>']),
|
||||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
adaptor = adaptors.RegexAdaptor({'regex': r'href="(.*?)"'})
|
||||
self.assertEqual(adaptor(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
|
||||
'<a href="das.biz">href="lelelel.net"</a>']),
|
||||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
|
||||
def test_unquote_all(self):
|
||||
self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
self.assertEqual(adaptors.UnquoteAdaptor({'keep': []})([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_unquote(self):
|
||||
self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
self.assertEqual(adaptors.UnquoteAdaptor()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_remove_tags(self):
|
||||
test_data = ['<a href="lala">adsaas<br /></a>', '<div id="1"><table>dsadasf</table></div>']
|
||||
self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
|
||||
self.assertEqual(adaptors.RemoveTagsAdaptor()(test_data), ['adsaas', 'dsadasf'])
|
||||
|
||||
def test_remove_root(self):
|
||||
self.assertEqual(adaptors.remove_root(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
|
||||
self.assertEqual(adaptors.RemoveRootAdaptor()(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
|
||||
['lallaa<a href="coso">dsfsdfds</a>pepepep<br />'])
|
||||
|
||||
def test_remove_multispaces(self):
|
||||
self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']),
|
||||
self.assertEqual(adaptors.CleanSpacesAdaptor()([' hello, whats up?', 'testing testingtesting testing']),
|
||||
[' hello, whats up?', 'testing testingtesting testing'])
|
||||
|
||||
def test_strip(self):
|
||||
self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
|
||||
self.assertEqual(adaptors.StripAdaptor()([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
|
||||
['hi there, sweety ;D', 'I CAN HAZ TEST??'])
|
||||
|
||||
def test_drop_empty_elements(self):
|
||||
self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']),
|
||||
self.assertEqual(adaptors.DropEmptyAdaptor()([1, 2, None, 5, None, 6, None, 'hi']),
|
||||
[1, 2, 5, 6, 'hi'])
|
||||
|
||||
def test_delist(self):
|
||||
self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
self.assertEqual(adaptors.DelistAdaptor()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
'hi there fellas. this is my test.')
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue