diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py
index bf57f5787..9004fc8d3 100644
--- a/scrapy/trunk/scrapy/item/adaptors.py
+++ b/scrapy/trunk/scrapy/item/adaptors.py
@@ -57,11 +57,13 @@ def extract(location):
"""
if not location:
return []
- elif isinstance(location, (list, tuple)):
+ elif isinstance(location, XPathSelectorList):
return flatten([extract(o) for o in location])
elif isinstance(location, XPathSelector):
return location.extract()
- elif isinstance(location, str):
+ elif isinstance(location, (list, tuple)):
+ return flatten(location)
+ elif isinstance(location, basestring):
return [location]
def _absolutize_links(rel_links, current_url, base_url):
@@ -103,6 +105,8 @@ def extract_links(locations):
if len(children) > 1:
ret.extend(selector.x('.//@href'))
ret.extend(selector.x('.//@src'))
+ elif len(children) == 1 and children[0].xmlNode.name == 'img':
+ ret.extend(children.x('@src'))
else:
ret.extend(selector.x('@href'))
elif selector.xmlNode.name == 'img':
@@ -113,7 +117,7 @@ def extract_links(locations):
elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
ret.append(selector)
ret = [selector.extract() for selector in ret]
- current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)/', locations[0].response.url).groups()
+ current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups()
ret = _absolutize_links(ret, current_url, base_url)
return ret
@@ -141,7 +145,7 @@ def regex(expr):
if isinstance(value, (XPathSelector, XPathSelectorList)):
return value.re(expr)
elif isinstance(value, list) and value:
- return extract_regex(expr, value, 'utf-8')
+ return flatten([extract_regex(expr, string, 'utf-8') for string in value])
return value
return _regex
@@ -217,16 +221,18 @@ def url_pipeline():
drop_empty_elements,
]
-def list_pipeline():
- return [ extract,
- unique,
- to_unicode,
- drop_empty_elements,
- unquote,
- remove_tags,
- remove_root,
- strip,
- ]
+def list_pipeline(extract=True):
+ pipe = []
+ if extract:
+ pipe.append(extract)
+ return pipe.extend([ unique,
+ to_unicode,
+ drop_empty_elements,
+ unquote,
+ remove_tags,
+ remove_root,
+ strip,
+ ])
def list_join_pipeline(delimiter='\t'):
return list_pipeline() + [delimiter.join]
diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py
new file mode 100644
index 000000000..df1627b5a
--- /dev/null
+++ b/scrapy/trunk/scrapy/tests/test_adaptors.py
@@ -0,0 +1,84 @@
+# -*- coding: utf8 -*-
+import unittest
+from scrapy.item import adaptors
+from scrapy.xpath.selector import XmlXPathSelector
+from scrapy.http import Response, ResponseBody
+
+class AdaptorsTestCase(unittest.TestCase):
+ def test_extract(self):
+ sample_xsel = XmlXPathSelector(text='foobartest')
+ self.assertEqual(adaptors.extract(sample_xsel.x('/')),
+ ['foobartest'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')),
+ ['foobar', 'test'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')),
+ ['foo', 'bar'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('//text()')),
+ ['foo', 'bar', 'test'])
+ self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag'])
+
+ def test_extract_links(self):
+ test_data = """
+
+ """
+ sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data))
+ sample_xsel = XmlXPathSelector(sample_response)
+ self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')),
+ [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
+ u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html'])
+ self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')),
+ [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html',
+ u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg'])
+ self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')),
+ [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html'])
+
+ def test_to_unicode(self):
+ self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']),
+ [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9'])
+
+ def test_regex(self):
+ func = adaptors.regex(r'href="(.*?)"')
+ self.assertEqual(func(['dsa',
+ 'href="lelelel.net"']),
+ ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
+
+ def test_unquote_all(self):
+ self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&'])
+
+ def test_unquote(self):
+ self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
+
+ def test_remove_tags(self):
+ test_data = ['adsaas
', '']
+ self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf'])
+
+ def test_remove_root(self):
+ self.assertEqual(adaptors.remove_root(['']),
+ ['lallaadsfsdfdspepepep
'])
+
+ def test_remove_multispaces(self):
+ self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']),
+ [' hello, whats up?', 'testing testingtesting testing'])
+
+ def test_strip(self):
+ self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']),
+ ['hi there, sweety ;D', 'I CAN HAZ TEST??'])
+
+ def test_drop_empty_elements(self):
+ self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']),
+ [1, 2, 5, 6, 'hi'])
+
+ def test_delist(self):
+ self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
+ 'hi there fellas. this is my test.')
+
+
+ something1
+ something2
+