mirror of https://github.com/scrapy/scrapy.git
removed obsolete adaptors code
This commit is contained in:
Pablo Hoffman
parent
e6dd4d0955
commit
6452e1934c
|
|
@ -1,5 +0,0 @@
|
|||
from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks
|
||||
from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, replace_escape, unquote
|
||||
from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex
|
||||
from scrapy.contrib_exp.adaptors.date import to_date
|
||||
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
import time
|
||||
|
||||
def to_date(format):
|
||||
"""
|
||||
Converts a date string to a YYYY-MM-DD one suitable for DateField
|
||||
|
||||
format is the format string passed to time.strptime
|
||||
|
||||
Input: string/unicode
|
||||
Output: string
|
||||
"""
|
||||
def _to_date(value):
|
||||
return time.strftime('%Y-%m-%d', time.strptime(value, format))
|
||||
return _to_date
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
"""
|
||||
Adaptors related with extraction of data
|
||||
"""
|
||||
|
||||
from scrapy.utils.url import urljoin_rfc
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.utils.python import unicode_to_str
|
||||
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
|
||||
from scrapy.contrib.linkextractors.image import HTMLImageLinkExtractor
|
||||
|
||||
def extract(location, adaptor_args=None):
|
||||
"""
|
||||
This adaptor tries to extract data from the given locations.
|
||||
Any XPathSelector in it will be extracted, and any other data
|
||||
will be added as-is to the result.
|
||||
|
||||
If an XPathSelector is a text/cdata node, and `use_unquote`
|
||||
is True, that selector will be extracted using the `extract_unquoted`
|
||||
method; otherwise, the `extract` method will be used.
|
||||
|
||||
Input: anything
|
||||
Output: tuple of extracted selectors plus anything else in the input
|
||||
"""
|
||||
use_unquote = adaptor_args.get('use_unquote', True) if adaptor_args else True
|
||||
|
||||
if isinstance(location, XPathSelectorList):
|
||||
ret = location.extract()
|
||||
elif isinstance(location, XPathSelector):
|
||||
if location.xmlNode.type in ('text', 'cdata') and use_unquote:
|
||||
ret = location.extract_unquoted()
|
||||
else:
|
||||
ret = location.extract()
|
||||
ret = ret if hasattr(ret, '__iter__') else [ret]
|
||||
elif isinstance(location, list):
|
||||
ret = tuple(location)
|
||||
else:
|
||||
ret = tuple([location])
|
||||
|
||||
return tuple(x for x in ret if x is not None)
|
||||
|
||||
class ExtractImageLinks(object):
|
||||
"""
|
||||
This adaptor may receive either XPathSelectors pointing to
|
||||
the desired locations for finding image urls, or just a list of
|
||||
XPath expressions (which will be turned into selectors anyway).
|
||||
|
||||
Input: XPathSelector, XPathSelectorList, iterable
|
||||
Output: tuple of urls (strings)
|
||||
"""
|
||||
def __init__(self, response=None, canonicalize=True):
|
||||
self.response = response
|
||||
self.canonicalize = canonicalize
|
||||
|
||||
def __call__(self, locations):
|
||||
if not locations:
|
||||
return tuple()
|
||||
elif not hasattr(locations, '__iter__'):
|
||||
locations = [locations]
|
||||
|
||||
selectors, raw_links = [], []
|
||||
for location in locations:
|
||||
if isinstance(location, (XPathSelector, XPathSelectorList)):
|
||||
selectors.append(location)
|
||||
else:
|
||||
raw_links.append(location)
|
||||
|
||||
if raw_links:
|
||||
base_url = get_base_url(self.response)
|
||||
raw_links = [urljoin_rfc(base_url, unicode_to_str(rel_url), self.response.encoding) for rel_url in raw_links]
|
||||
|
||||
lx = HTMLImageLinkExtractor(locations=selectors, canonicalize=self.canonicalize)
|
||||
urls = map(lambda link: link.url, lx.extract_links(self.response))
|
||||
return tuple(urls + raw_links)
|
||||
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import re
|
||||
from scrapy.utils.markup import replace_tags, replace_escape_chars, unquote_markup
|
||||
from scrapy.utils.python import str_to_unicode
|
||||
|
||||
def remove_tags(tags=()):
|
||||
"""
|
||||
Factory that returns an adaptor for removing each
|
||||
tag in the `tags` parameter found in the given value.
|
||||
If no `tags` are specified, all of them are removed.
|
||||
|
||||
Input: string/unicode
|
||||
Output: unicode
|
||||
"""
|
||||
def _remove_tags(value):
|
||||
return replace_tags(value, tags)
|
||||
return _remove_tags
|
||||
|
||||
_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
|
||||
def remove_root(value):
|
||||
"""
|
||||
This adaptor removes the root tag of the given string/unicode,
|
||||
if it's found.
|
||||
|
||||
Input: string/unicode
|
||||
Output: unicode
|
||||
"""
|
||||
m = _remove_root_re.search(value)
|
||||
if m:
|
||||
value = m.group(1)
|
||||
return str_to_unicode(value)
|
||||
|
||||
def replace_escape(which_ones=('\n', '\t', '\r'), replace_by=u''):
|
||||
"""
|
||||
Factory that returns an adaptor for removing/replacing each escape
|
||||
character in the `wich_ones` parameter found in the given value.
|
||||
|
||||
If `replace_by` is given, escape characters are replaced by that
|
||||
text, else they're removed.
|
||||
|
||||
Input: string/unicode
|
||||
Output: unicode
|
||||
|
||||
"""
|
||||
def _replace_escape(value):
|
||||
return replace_escape_chars(value, which_ones, replace_by)
|
||||
return _replace_escape
|
||||
|
||||
def unquote(keep=None):
|
||||
"""
|
||||
This factory returns an adaptor that
|
||||
receives a string or unicode, removes all of the
|
||||
CDATAs and entities (except the ones in CDATAs, and the ones
|
||||
you specify in the `keep` parameter) and then, returns a new
|
||||
string or unicode.
|
||||
|
||||
Input: string/unicode
|
||||
Output: string/unicode
|
||||
"""
|
||||
default_keep = [] if keep is None else keep
|
||||
|
||||
def unquote(value, adaptor_args):
|
||||
keep = adaptor_args.get('keep_entities', default_keep)
|
||||
return unquote_markup(value, keep=keep)
|
||||
return unquote
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.python import flatten, str_to_unicode, unicode_to_str
|
||||
from scrapy.contrib.item.adaptors import adaptize
|
||||
|
||||
def to_unicode(value, adaptor_args):
|
||||
"""
|
||||
Receives a string and converts it to unicode
|
||||
using the given encoding (if specified, else utf-8 is used)
|
||||
and returns a new unicode object.
|
||||
E.g:
|
||||
>> to_unicode('it costs 20\xe2\x82\xac, or 30\xc2\xa3')
|
||||
[u'it costs 20\u20ac, or 30\xa3']
|
||||
|
||||
Input: string
|
||||
Output: unicode
|
||||
"""
|
||||
if not isinstance(value, basestring):
|
||||
value = str(value)
|
||||
return str_to_unicode(value, adaptor_args.get('encoding', 'utf-8'))
|
||||
|
||||
_clean_spaces_re = re.compile("\s+", re.U)
|
||||
def clean_spaces(value):
|
||||
"""
|
||||
Converts multispaces into single spaces for the given string.
|
||||
E.g:
|
||||
>> clean_spaces(u'Hello sir')
|
||||
u'Hello sir'
|
||||
|
||||
Input: string/unicode
|
||||
Output: unicode
|
||||
"""
|
||||
return _clean_spaces_re.sub(' ', str_to_unicode(value))
|
||||
|
||||
def strip(value):
|
||||
"""
|
||||
Removes any spaces at both the start and the ending
|
||||
of the provided string.
|
||||
E.g:
|
||||
>> strip(' hi buddies ')
|
||||
u'hi buddies'
|
||||
|
||||
Input: string/unicode
|
||||
Output: string/unicode
|
||||
"""
|
||||
return value.strip()
|
||||
|
||||
def drop_empty(value):
|
||||
"""
|
||||
Removes any index that evaluates to None
|
||||
from the provided iterable.
|
||||
E.g:
|
||||
>> drop_empty([0, 'this', None, 'is', False, 'an example'])
|
||||
['this', 'is', 'an example']
|
||||
|
||||
Input: iterable
|
||||
Output: list
|
||||
"""
|
||||
return [ v for v in value if v ]
|
||||
|
||||
def delist(delimiter=''):
|
||||
"""
|
||||
This factory returns and adaptor that joins
|
||||
an iterable with the specified delimiter.
|
||||
|
||||
Input: iterable of strings/unicodes
|
||||
Output: string/unicode
|
||||
"""
|
||||
def delist(value, adaptor_args):
|
||||
delim = adaptor_args.get('join_delimiter', delimiter)
|
||||
return delim.join(value)
|
||||
return delist
|
||||
|
||||
class Regex(object):
|
||||
"""
|
||||
This adaptor must receive either a list of strings or an XPathSelector
|
||||
and return a new list with the matches of the given strings with the given regular
|
||||
expression (which is passed by a keyword argument, and is mandatory for this adaptor).
|
||||
|
||||
Input: XPathSelector, XPathSelectorList, iterable
|
||||
Output: list of unicodes
|
||||
"""
|
||||
def __init__(self, regex=r''):
|
||||
self.regex = regex
|
||||
|
||||
def __call__(self, value):
|
||||
if self.regex:
|
||||
if isinstance(value, (XPathSelector, XPathSelectorList)):
|
||||
return value.re(self.regex)
|
||||
elif hasattr(value, '__iter__'):
|
||||
return flatten([extract_regex(self.regex, string, 'utf-8') for string in value])
|
||||
return value
|
||||
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
||||
|
||||
def adaptize(func):
|
||||
func_args = get_func_args(func)
|
||||
if 'adaptor_args' in func_args:
|
||||
return func
|
||||
|
||||
def _adaptor(value, adaptor_args):
|
||||
return func(value)
|
||||
return _adaptor
|
||||
|
||||
|
||||
def IDENTITY(v):
|
||||
return v
|
||||
|
||||
|
||||
class ItemAdaptorMeta(type):
|
||||
def __new__(meta, class_name, bases, attrs):
|
||||
da = attrs.get('default_adaptor')
|
||||
if da:
|
||||
attrs['default_adaptor'] = staticmethod(adaptize(da))
|
||||
|
||||
cls = type.__new__(meta, class_name, bases, attrs)
|
||||
|
||||
cls._field_adaptors = cls._field_adaptors.copy()
|
||||
|
||||
if cls.item_class:
|
||||
for item_field in cls.item_class.fields:
|
||||
if item_field in attrs:
|
||||
adaptor = adaptize(attrs[item_field])
|
||||
cls._field_adaptors[item_field] = adaptor
|
||||
setattr(cls, item_field, staticmethod(adaptor))
|
||||
return cls
|
||||
|
||||
def __getattr__(cls, name):
|
||||
if name in cls.item_class.fields:
|
||||
return cls.default_adaptor
|
||||
raise AttributeError(name)
|
||||
|
||||
|
||||
class ItemAdaptor(object):
|
||||
__metaclass__ = ItemAdaptorMeta
|
||||
|
||||
item_class = None
|
||||
default_adaptor = IDENTITY
|
||||
|
||||
_field_adaptors = {}
|
||||
|
||||
def __init__(self, response=None, item=None):
|
||||
self.item_instance = item if item else self.item_class()
|
||||
self._response = response
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if (name.startswith('_') or name == 'item_instance' \
|
||||
or name == 'default_adaptor'):
|
||||
return object.__setattr__(self, name, value)
|
||||
|
||||
fa = self._field_adaptors.get(name, self.default_adaptor)
|
||||
|
||||
adaptor_args = {'response': self._response, 'item': self.item_instance}
|
||||
ovalue = fa(value, adaptor_args=adaptor_args)
|
||||
self.item_instance[name] = ovalue
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if (name.startswith('_') or name.startswith('item_') \
|
||||
or name == 'default_adaptor'):
|
||||
return object.__getattribute__(self, name)
|
||||
|
||||
return self.item_instance[name]
|
||||
|
||||
|
||||
def adaptor(*funcs, **default_adaptor_args):
|
||||
"""A pipe adaptor implementing the tree adaption logic
|
||||
|
||||
It takes multiples unnamed arguments used as functions of the pipe, and
|
||||
keywords used as adaptor_args to be passed to functions that supports it
|
||||
|
||||
If an adaptor function returns a list of values, each value is used as
|
||||
input for next adaptor function
|
||||
|
||||
Always returns a list of values
|
||||
"""
|
||||
|
||||
_funcs = []
|
||||
for func in funcs:
|
||||
accepts_args = 'adaptor_args' in get_func_args(func)
|
||||
_funcs.append((func, accepts_args))
|
||||
|
||||
def _adaptor(value, adaptor_args=None):
|
||||
values = arg_to_iter(value)
|
||||
aargs = default_adaptor_args
|
||||
if adaptor_args:
|
||||
aargs = aargs.copy()
|
||||
aargs.update(adaptor_args)
|
||||
pipe_kwargs = {'adaptor_args': aargs}
|
||||
for func, accepts_args in _funcs:
|
||||
next = []
|
||||
kwargs = pipe_kwargs if accepts_args else {}
|
||||
for val in values:
|
||||
val = func(val, **kwargs)
|
||||
if hasattr(val, '__iter__'):
|
||||
next.extend(val)
|
||||
elif val is not None:
|
||||
next.append(val)
|
||||
values = next
|
||||
return list(values)
|
||||
|
||||
return _adaptor
|
||||
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import unittest
|
||||
import re
|
||||
|
||||
from scrapy.contrib.item.adaptors import AdaptorPipe
|
||||
from scrapy.contrib_exp import adaptors
|
||||
from scrapy.http import HtmlResponse, Headers
|
||||
from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector
|
||||
from scrapy.tests import get_testdata
|
||||
|
||||
class AdaptorPipeTestCase(unittest.TestCase):
|
||||
def test_pipe_init(self):
|
||||
self.assertRaises(TypeError, AdaptorPipe, [adaptors.extract, 'a string'])
|
||||
|
||||
def test_adaptor_args(self):
|
||||
def sample_adaptor(value, adaptor_args):
|
||||
'''Dummy adaptor that joins the received value with the given string'''
|
||||
sample_text = adaptor_args.get('sample_arg', 'sample text 1')
|
||||
return '%s "%s"' % (value, sample_text)
|
||||
|
||||
sample_value = 'hi, this is my text:'
|
||||
sample_pipe = AdaptorPipe([sample_adaptor])
|
||||
self.assertEqual(sample_pipe(sample_value), ('hi, this is my text: "sample text 1"', ))
|
||||
self.assertEqual(sample_pipe(sample_value, {'sample_arg': 'foobarfoobar'}),
|
||||
('hi, this is my text: "foobarfoobar"', ))
|
||||
|
||||
def test_add(self):
|
||||
pipe1 = AdaptorPipe([adaptors.extract])
|
||||
pipe2 = [adaptors.remove_tags]
|
||||
pipe3 = (adaptors.remove_root, )
|
||||
sample_callable = dir
|
||||
|
||||
self.assertTrue(isinstance(pipe1 + pipe1, AdaptorPipe))
|
||||
self.assertTrue(isinstance(pipe1 + pipe2, AdaptorPipe))
|
||||
self.assertTrue(isinstance(pipe1 + pipe3, AdaptorPipe))
|
||||
self.assertTrue(isinstance(pipe1 + sample_callable, AdaptorPipe))
|
||||
|
||||
class AdaptorsTestCase(unittest.TestCase):
|
||||
|
||||
def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector):
|
||||
body = get_testdata('adaptors', sample_filename)
|
||||
response = HtmlResponse(url=url, headers=Headers(headers), status=200, body=body)
|
||||
return selector(response)
|
||||
|
||||
def test_extract(self):
|
||||
def check_extractor(x, pound=True, euro=True):
|
||||
poundre = re.compile(r'<span class="pound" .*?>(.*?)</span>')
|
||||
eurore = re.compile(r'<span class="euro" .*?>(.*?)</span>')
|
||||
|
||||
if pound:
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/text()")), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/@value")), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.re(poundre)), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/text()")), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/@value")), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/text()")), (u'\xa3', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/@value")), (u'\xa3', ))
|
||||
if euro:
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/text()")), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/@value")), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.re(eurore)), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/text()")), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/@value")), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/text()")), (u'\u20ac', ))
|
||||
self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/@value")), (u'\u20ac', ))
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/utf8',
|
||||
'enc-utf8.html',
|
||||
{})
|
||||
check_extractor(x)
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/latin1',
|
||||
'enc-latin1.html',
|
||||
{})
|
||||
check_extractor(x, euro=False)
|
||||
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/cp1252',
|
||||
'enc-cp1252.html',
|
||||
{})
|
||||
check_extractor(x)
|
||||
|
||||
# HTTP utf-8 | Meta latin1 | Content ascii | using entities
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/ascii',
|
||||
'enc-ascii.html',
|
||||
{'Content-Type': ['text/html; charset=utf-8']})
|
||||
check_extractor(x, pound=False, euro=False)
|
||||
|
||||
# Test for inconsistencies between HTTP header encoding and
|
||||
# META header encoding. It must prefer HTTP header like browsers do
|
||||
x = self.get_selector('example.com',
|
||||
'http://www.example.com/test/utf8-meta-latin1',
|
||||
'enc-utf8-meta-latin1.html',
|
||||
{'Content-Type': ['text/html; charset=utf-8']})
|
||||
check_extractor(x)
|
||||
|
||||
|
||||
def test_extract_unquoted(self):
|
||||
x = self.get_selector('example.com', 'http://www.example.com/test_unquoted', 'extr_unquoted.xml', selector=XmlXPathSelector)
|
||||
|
||||
# test unquoting
|
||||
self.assertEqual(adaptors.extract(x.x('//tag1/text()[2]')[0]), (u'more test text & >', ))
|
||||
self.assertEqual(adaptors.extract(x.x('//tag2/text()[1]')[0]), (u'blaheawfds<', ))
|
||||
|
||||
# test without unquoting
|
||||
self.assertEqual(adaptors.extract(x.x('//tag1/text()'), {'use_unquote': False}),
|
||||
(u'test text & &', u'<![CDATA[more test text & >]]>', u'blah&blah'))
|
||||
self.assertEqual(adaptors.extract(x.x('//tag2/text()'), {'use_unquote': False}), (u'blaheawfds<', ))
|
||||
|
||||
def test_extract_image_links(self):
|
||||
test_data = """<html><body>
|
||||
<div>
|
||||
<a href="lala1/lala1.html">lala1</a>
|
||||
<a href="/lala2.html">lala2</a>
|
||||
<a href="http://foobar.com/pepepe/papapa/lala3.html">lala3</a>
|
||||
<a href="lala4.html"><img src="/imgs/lala4.jpg" /></a>
|
||||
<a onclick="javascript: opensomething('/my_html1.html');">something1</a>
|
||||
<a onclick="javascript: opensomething('dummy/my_html2.html');">something2</a>
|
||||
</div>
|
||||
</body></html>"""
|
||||
sample_response = HtmlResponse('http://foobar.com/dummy', body=test_data)
|
||||
sample_selector = HtmlXPathSelector(sample_response)
|
||||
sample_adaptor = adaptors.ExtractImageLinks(response=sample_response)
|
||||
|
||||
self.assertEqual(sample_adaptor(None), tuple())
|
||||
self.assertEqual(sample_adaptor([]), tuple())
|
||||
self.assertEqual(sample_adaptor([sample_selector.x('//@href'), 'http://foobar.com/my_image.gif']), (
|
||||
'http://foobar.com/lala1/lala1.html',
|
||||
'http://foobar.com/lala2.html',
|
||||
'http://foobar.com/pepepe/papapa/lala3.html',
|
||||
'http://foobar.com/lala4.html',
|
||||
'http://foobar.com/my_image.gif',
|
||||
))
|
||||
self.assertEqual(sample_adaptor(sample_selector.x('//a')), (
|
||||
'http://foobar.com/lala1/lala1.html',
|
||||
'http://foobar.com/lala2.html',
|
||||
'http://foobar.com/pepepe/papapa/lala3.html',
|
||||
'http://foobar.com/imgs/lala4.jpg',
|
||||
))
|
||||
self.assertEqual(sample_adaptor(sample_selector.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')), (
|
||||
'http://foobar.com/my_html1.html',
|
||||
'http://foobar.com/dummy/my_html2.html'
|
||||
))
|
||||
|
||||
def test_to_unicode(self):
|
||||
self.assertEqual(adaptors.to_unicode('lelelulu\xc3\xb1', {}), u'lelelulu\xf1')
|
||||
self.assertEqual(adaptors.to_unicode(1, {}), u'1')
|
||||
self.assertEqual(adaptors.to_unicode('\xc3\xa1\xc3\xa9', {'encoding': 'utf-8'}), u'\xe1\xe9')
|
||||
self.assertEqual(adaptors.to_unicode('\xf1', {'encoding': 'latin1'}), u'\xf1')
|
||||
|
||||
def test_regex(self):
|
||||
adaptor = adaptors.Regex(regex=r'href="(.*?)"')
|
||||
self.assertEqual(adaptor(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
|
||||
'<a href="das.biz">href="lelelel.net"</a>']),
|
||||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
|
||||
def test_unquote_all(self):
|
||||
unquote = adaptors.unquote()
|
||||
self.assertEqual(unquote(u'hello©&welcome<br />&', {}),
|
||||
u'hello\xa9&welcome<br />&')
|
||||
|
||||
def test_unquote(self):
|
||||
unquote = adaptors.unquote(keep=['amp', 'lt'])
|
||||
self.assertEqual(unquote(u'hello©&welcome<br />&', {}),
|
||||
u'hello\xa9&welcome<br />&')
|
||||
|
||||
def test_remove_tags(self):
|
||||
self.assertEqual(adaptors.remove_tags()('<a href="lala">adsaas<br /></a>'), 'adsaas')
|
||||
self.assertEqual(adaptors.remove_tags()('<div id="1"><table>dsadasf</table></div>'), 'dsadasf')
|
||||
|
||||
def test_remove_root(self):
|
||||
self.assertEqual(adaptors.remove_root('<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>'),
|
||||
'lallaa<a href="coso">dsfsdfds</a>pepepep<br />')
|
||||
|
||||
def test_remove_multispaces(self):
|
||||
self.assertEqual(adaptors.clean_spaces(' hello, whats up?'), ' hello, whats up?')
|
||||
|
||||
def test_strip(self):
|
||||
self.assertEqual(adaptors.strip(' hello there, this is my test '),
|
||||
'hello there, this is my test')
|
||||
|
||||
def test_drop_empty_elements(self):
|
||||
self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']),
|
||||
[1, 2, 5, 6, 'hi'])
|
||||
|
||||
def test_delist(self):
|
||||
delist = adaptors.delist(' ')
|
||||
self.assertEqual(delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.'], {}),
|
||||
'hi there fellas. this is my test.')
|
||||
self.assertEqual(delist(['hi', 'there', 'fellas,', 'this', 'is', 'my', 'test.'], {'join_delimiter': '.'}),
|
||||
'hi.there.fellas,.this.is.my.test.')
|
||||
|
||||
|
||||
|
|
@ -2,8 +2,6 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.contrib.item import RobustScrapedItem
|
||||
from scrapy.contrib.item.adaptors import AdaptorPipe
|
||||
from scrapy.contrib_exp import adaptors
|
||||
|
||||
class MyItem(RobustScrapedItem):
|
||||
ATTRIBUTES = {
|
||||
|
|
@ -55,37 +53,3 @@ class RobustScrapedItemTestCase(unittest.TestCase):
|
|||
self.item.attribute('children', 'Johnny', 'Rodrigo', add=True)
|
||||
self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy', 'Johnny', 'Rodrigo'])
|
||||
|
||||
def test_set_adaptors(self):
|
||||
self.assertEqual(self.item._adaptors_dict, {})
|
||||
|
||||
delist = adaptors.delist()
|
||||
self.item.set_adaptors({'name': [adaptors.extract, delist]})
|
||||
self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe))
|
||||
self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract")
|
||||
self.assertEqual(self.item._adaptors_dict['name'][1].name, "delist")
|
||||
|
||||
self.item.set_adaptors({'description': [adaptors.extract]})
|
||||
self.assertEqual(self.item._adaptors_dict['description'][0].name, "extract")
|
||||
|
||||
def test_set_attrib_adaptors(self):
|
||||
self.assertEqual(self.item._adaptors_dict, {})
|
||||
|
||||
self.item.set_attrib_adaptors('name', [adaptors.extract, adaptors.strip])
|
||||
self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe))
|
||||
self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract")
|
||||
self.assertEqual(self.item._adaptors_dict['name'][1].name, "strip")
|
||||
|
||||
unquote = adaptors.unquote()
|
||||
self.item.set_attrib_adaptors('name', [adaptors.extract, unquote])
|
||||
self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe))
|
||||
self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract")
|
||||
self.assertEqual(self.item._adaptors_dict['name'][1].name, "unquote")
|
||||
|
||||
def test_add_adaptor(self):
|
||||
self.assertEqual(self.item._adaptors_dict, {})
|
||||
|
||||
self.item.add_adaptor('name', adaptors.strip)
|
||||
self.assertEqual(self.item._adaptors_dict['name'][0].name, "strip")
|
||||
self.item.add_adaptor('name', adaptors.extract, position=0)
|
||||
self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract")
|
||||
self.assertEqual(self.item._adaptors_dict['name'][1].name, "strip")
|
||||
|
|
|
|||
Loading…
Reference in New Issue