From 3433272d71db44a95ce49a7190f79067eaeaff61 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 16 Oct 2008 13:29:23 +0000 Subject: [PATCH] - Added adaptors tests - Fixed some small bugs on a few adaptors --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40316 --- scrapy/trunk/scrapy/item/adaptors.py | 34 +++++---- scrapy/trunk/scrapy/tests/test_adaptors.py | 84 ++++++++++++++++++++++ 2 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 scrapy/trunk/scrapy/tests/test_adaptors.py diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index bf57f5787..9004fc8d3 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -57,11 +57,13 @@ def extract(location): """ if not location: return [] - elif isinstance(location, (list, tuple)): + elif isinstance(location, XPathSelectorList): return flatten([extract(o) for o in location]) elif isinstance(location, XPathSelector): return location.extract() - elif isinstance(location, str): + elif isinstance(location, (list, tuple)): + return flatten(location) + elif isinstance(location, basestring): return [location] def _absolutize_links(rel_links, current_url, base_url): @@ -103,6 +105,8 @@ def extract_links(locations): if len(children) > 1: ret.extend(selector.x('.//@href')) ret.extend(selector.x('.//@src')) + elif len(children) == 1 and children[0].xmlNode.name == 'img': + ret.extend(children.x('@src')) else: ret.extend(selector.x('@href')) elif selector.xmlNode.name == 'img': @@ -113,7 +117,7 @@ def extract_links(locations): elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: ret.append(selector) ret = [selector.extract() for selector in ret] - current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)/', locations[0].response.url).groups() + current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups() ret = _absolutize_links(ret, current_url, base_url) return ret @@ -141,7 +145,7 @@ def regex(expr): if isinstance(value, (XPathSelector, XPathSelectorList)): return value.re(expr) elif isinstance(value, list) and value: - return extract_regex(expr, value, 'utf-8') + return flatten([extract_regex(expr, string, 'utf-8') for string in value]) return value return _regex @@ -217,16 +221,18 @@ def url_pipeline(): drop_empty_elements, ] -def list_pipeline(): - return [ extract, - unique, - to_unicode, - drop_empty_elements, - unquote, - remove_tags, - remove_root, - strip, - ] +def list_pipeline(extract=True): + pipe = [] + if extract: + pipe.append(extract) + return pipe.extend([ unique, + to_unicode, + drop_empty_elements, + unquote, + remove_tags, + remove_root, + strip, + ]) def list_join_pipeline(delimiter='\t'): return list_pipeline() + [delimiter.join] diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py new file mode 100644 index 000000000..df1627b5a --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_adaptors.py @@ -0,0 +1,84 @@ +# -*- coding: utf8 -*- +import unittest +from scrapy.item import adaptors +from scrapy.xpath.selector import XmlXPathSelector +from scrapy.http import Response, ResponseBody + +class AdaptorsTestCase(unittest.TestCase): + def test_extract(self): + sample_xsel = XmlXPathSelector(text='foobartest') + self.assertEqual(adaptors.extract(sample_xsel.x('/')), + ['foobartest']) + self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')), + ['foobar', 'test']) + self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), + ['foo', 'bar']) + self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), + ['foo', 'bar', 'test']) + self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag']) + + def test_extract_links(self): + test_data = """ +
+ lala1 + lala2 + lala3 + + something1 + something2 +
+ """ + sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data)) + sample_xsel = XmlXPathSelector(sample_response) + self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')), + [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', + u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html']) + self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')), + [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', + u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg']) + self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')), + [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html']) + + def test_to_unicode(self): + self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']), + [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9']) + + def test_regex(self): + func = adaptors.regex(r'href="(.*?)"') + self.assertEqual(func(['dsa', + 'href="lelelel.net"']), + ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) + + def test_unquote_all(self): + self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&']) + + def test_unquote(self): + self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) + + def test_remove_tags(self): + test_data = ['adsaas
', '
dsadasf
'] + self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf']) + + def test_remove_root(self): + self.assertEqual(adaptors.remove_root(['
lallaadsfsdfdspepepep
']), + ['lallaadsfsdfdspepepep
']) + + def test_remove_multispaces(self): + self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']), + [' hello, whats up?', 'testing testingtesting testing']) + + def test_strip(self): + self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), + ['hi there, sweety ;D', 'I CAN HAZ TEST??']) + + def test_drop_empty_elements(self): + self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']), + [1, 2, 5, 6, 'hi']) + + def test_delist(self): + self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), + 'hi there fellas. this is my test.') + +