mirror of https://github.com/scrapy/scrapy.git
- Added adaptors tests
- Fixed some small bugs on a few adaptors --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40316
This commit is contained in:
elpolilla
parent
0ed5978abf
commit
3433272d71
|
|
@ -57,11 +57,13 @@ def extract(location):
|
|||
"""
|
||||
if not location:
|
||||
return []
|
||||
elif isinstance(location, (list, tuple)):
|
||||
elif isinstance(location, XPathSelectorList):
|
||||
return flatten([extract(o) for o in location])
|
||||
elif isinstance(location, XPathSelector):
|
||||
return location.extract()
|
||||
elif isinstance(location, str):
|
||||
elif isinstance(location, (list, tuple)):
|
||||
return flatten(location)
|
||||
elif isinstance(location, basestring):
|
||||
return [location]
|
||||
|
||||
def _absolutize_links(rel_links, current_url, base_url):
|
||||
|
|
@ -103,6 +105,8 @@ def extract_links(locations):
|
|||
if len(children) > 1:
|
||||
ret.extend(selector.x('.//@href'))
|
||||
ret.extend(selector.x('.//@src'))
|
||||
elif len(children) == 1 and children[0].xmlNode.name == 'img':
|
||||
ret.extend(children.x('@src'))
|
||||
else:
|
||||
ret.extend(selector.x('@href'))
|
||||
elif selector.xmlNode.name == 'img':
|
||||
|
|
@ -113,7 +117,7 @@ def extract_links(locations):
|
|||
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
|
||||
ret.append(selector)
|
||||
ret = [selector.extract() for selector in ret]
|
||||
current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)/', locations[0].response.url).groups()
|
||||
current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups()
|
||||
ret = _absolutize_links(ret, current_url, base_url)
|
||||
return ret
|
||||
|
||||
|
|
@ -141,7 +145,7 @@ def regex(expr):
|
|||
if isinstance(value, (XPathSelector, XPathSelectorList)):
|
||||
return value.re(expr)
|
||||
elif isinstance(value, list) and value:
|
||||
return extract_regex(expr, value, 'utf-8')
|
||||
return flatten([extract_regex(expr, string, 'utf-8') for string in value])
|
||||
return value
|
||||
return _regex
|
||||
|
||||
|
|
@ -217,16 +221,18 @@ def url_pipeline():
|
|||
drop_empty_elements,
|
||||
]
|
||||
|
||||
def list_pipeline():
|
||||
return [ extract,
|
||||
unique,
|
||||
to_unicode,
|
||||
drop_empty_elements,
|
||||
unquote,
|
||||
remove_tags,
|
||||
remove_root,
|
||||
strip,
|
||||
]
|
||||
def list_pipeline(extract=True):
|
||||
pipe = []
|
||||
if extract:
|
||||
pipe.append(extract)
|
||||
return pipe.extend([ unique,
|
||||
to_unicode,
|
||||
drop_empty_elements,
|
||||
unquote,
|
||||
remove_tags,
|
||||
remove_root,
|
||||
strip,
|
||||
])
|
||||
|
||||
def list_join_pipeline(delimiter='\t'):
|
||||
return list_pipeline() + [delimiter.join]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import unittest
|
||||
from scrapy.item import adaptors
|
||||
from scrapy.xpath.selector import XmlXPathSelector
|
||||
from scrapy.http import Response, ResponseBody
|
||||
|
||||
class AdaptorsTestCase(unittest.TestCase):
|
||||
def test_extract(self):
|
||||
sample_xsel = XmlXPathSelector(text='<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>')
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('/')),
|
||||
['<xml id="2"><tag1>foo<tag2>bar</tag2></tag1><tag3 value="mytag">test</tag3></xml>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')),
|
||||
['<tag1>foo<tag2>bar</tag2></tag1>', '<tag3 value="mytag">test</tag3>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['<tag1>foo<tag2>bar</tag2></tag1>'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')),
|
||||
['foo', 'bar'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//text()')),
|
||||
['foo', 'bar', 'test'])
|
||||
self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag'])
|
||||
|
||||
def test_extract_links(self):
|
||||
test_data = """<html><body>
|
||||
<div>
|
||||
<a href="lala1.html">lala1</a>
|
||||
<a href="/lala2.html">lala2</a>
|
||||
<a href="http://foobar.com/lala3.html">lala3</a>
|
||||
<a href="lala4.html"><img src="lala4.jpg" /></a>
|
||||
<a onclick="javascript: opensomething('/my_html1.html');">something1</a>
|
||||
<a onclick="javascript: opensomething('my_html2.html');">something2</a>
|
||||
</div>
|
||||
</body></html>"""
|
||||
sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data))
|
||||
sample_xsel = XmlXPathSelector(sample_response)
|
||||
self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')),
|
||||
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
|
||||
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html'])
|
||||
self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')),
|
||||
[u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
|
||||
u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg'])
|
||||
self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')),
|
||||
[u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
|
||||
|
||||
def test_to_unicode(self):
|
||||
self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
|
||||
[u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
|
||||
|
||||
def test_regex(self):
|
||||
func = adaptors.regex(r'href="(.*?)"')
|
||||
self.assertEqual(func(['<a href="lala.com">dsa</a><a href="pepe.co.uk"></a>',
|
||||
'<a href="das.biz">href="lelelel.net"</a>']),
|
||||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
|
||||
def test_unquote_all(self):
|
||||
self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_unquote(self):
|
||||
self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_remove_tags(self):
|
||||
test_data = ['<a href="lala">adsaas<br /></a>', '<div id="1"><table>dsadasf</table></div>']
|
||||
self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
|
||||
|
||||
def test_remove_root(self):
|
||||
self.assertEqual(adaptors.remove_root(['<div>lallaa<a href="coso">dsfsdfds</a>pepepep<br /></div>']),
|
||||
['lallaa<a href="coso">dsfsdfds</a>pepepep<br />'])
|
||||
|
||||
def test_remove_multispaces(self):
|
||||
self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']),
|
||||
[' hello, whats up?', 'testing testingtesting testing'])
|
||||
|
||||
def test_strip(self):
|
||||
self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
|
||||
['hi there, sweety ;D', 'I CAN HAZ TEST??'])
|
||||
|
||||
def test_drop_empty_elements(self):
|
||||
self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']),
|
||||
[1, 2, 5, 6, 'hi'])
|
||||
|
||||
def test_delist(self):
|
||||
self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
'hi there fellas. this is my test.')
|
||||
|
||||
|
||||
Loading…
Reference in New Issue