RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TakeFirst doc says it returns first non-null/non-empty value, zero is a valid value. closes #59

This commit is contained in:
Daniel Graña 2013-01-08 15:47:33 -02:00
parent 672d09ea2e
commit 487299e068
2 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/loader/processor.py
View File

@ -13,7 +13,7 @@ class MapCompose(object):
def __init__(self, *functions, **default_loader_context):
self.functions = functions
self.default_loader_context = default_loader_context
def __call__(self, value, loader_context=None):
values = arg_to_iter(value)
if loader_context:
@ -35,7 +35,7 @@ class Compose(object):
self.functions = functions
self.stop_on_none = default_loader_context.get('stop_on_none', True)
self.default_loader_context = default_loader_context
def __call__(self, value, loader_context=None):
if loader_context:
context = MergeDict(loader_context, self.default_loader_context)
@ -53,7 +53,7 @@ class TakeFirst(object):
def __call__(self, values):
for value in values:
if value:
if value is not None and value != '':
return value

18
scrapy/tests/test_contrib_loader.py
View File

@ -7,33 +7,37 @@ from scrapy.item import Item, Field
from scrapy.selector import HtmlXPathSelector
from scrapy.http import HtmlResponse
# test items
# test items
class NameItem(Item):
name = Field()
class TestItem(NameItem):
url = Field()
summary = Field()
# test item loaders
# test item loaders
class NameItemLoader(ItemLoader):
default_item_class = TestItem
class TestItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title())
class DefaultedItemLoader(NameItemLoader):
default_input_processor = MapCompose(lambda v: v[:-1])
# test processors
# test processors
def processor_with_args(value, other=None, loader_context=None):
if 'key' in loader_context:
return loader_context['key']
return value
class ItemLoaderTest(unittest.TestCase):
def test_load_item_using_default_loader(self):
@ -96,7 +100,6 @@ class ItemLoaderTest(unittest.TestCase):
il.replace_value('name', u'name:bar', re=u'name:(.*)$')
self.assertEqual([u'bar'], il.get_collected_values('name'))
def test_iter_on_input_processor_input(self):
class NameFirstItemLoader(NameItemLoader):
name_in = TakeFirst()
@ -215,7 +218,6 @@ class ItemLoaderTest(unittest.TestCase):
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), u'Mar Ta')
def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
default_item_class = TestItem
@ -240,7 +242,6 @@ class ItemLoaderTest(unittest.TestCase):
assert 'ValueError' in s, s
assert expected_exc_str in s, s
def test_output_processor_using_classes(self):
il = TestItemLoader()
il.add_value('name', [u'mar', u'ta'])
@ -331,11 +332,13 @@ class ItemLoaderTest(unittest.TestCase):
item = il.load_item()
self.assertEqual(item['name'], u'Mart')
class ProcessorsTest(unittest.TestCase):
def test_take_first(self):
proc = TakeFirst()
self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
def test_identity(self):
proc = Identity()
@ -363,10 +366,12 @@ class ProcessorsTest(unittest.TestCase):
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
class TestXPathItemLoader(XPathItemLoader):
default_item_class = TestItem
name_in = MapCompose(lambda v: v.title())
class XPathItemLoaderTest(unittest.TestCase):
response = HtmlResponse(url="", body='<html><body><div id="id">marta</div><p>paragraph</p></body></html>')
@ -428,4 +433,3 @@ class XPathItemLoaderTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()