From efa08318be39ded91ce9ce8a868e4ac8e6e4db93 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 7 Aug 2009 03:48:42 -0300 Subject: [PATCH] renamed JoinStrings reducer to Join, accept item as first positional argument in ItemLoader constructor, removed expanders and reducers docstrings (will be moved to documentation) --- scrapy/newitem/loader/__init__.py | 8 ++++---- scrapy/newitem/loader/expanders.py | 20 +++----------------- scrapy/newitem/loader/reducers.py | 14 ++++---------- scrapy/tests/test_itemloader.py | 6 +++--- 4 files changed, 14 insertions(+), 34 deletions(-) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py index 1ab0fb476..09640aabb 100644 --- a/scrapy/newitem/loader/__init__.py +++ b/scrapy/newitem/loader/__init__.py @@ -11,10 +11,10 @@ class ItemLoader(object): default_expander = IdentityExpander() default_reducer = TakeFirst() - def __init__(self, **loader_args): - if 'item' not in loader_args: - loader_args['item'] = self.default_item_class() - self._item = loader_args['item'] + def __init__(self, item=None, **loader_args): + if not item: + item = self.default_item_class() + self._item = loader_args['item'] = item self._loader_args = loader_args self._values = defaultdict(list) diff --git a/scrapy/newitem/loader/expanders.py b/scrapy/newitem/loader/expanders.py index 33fc0f833..3e2cb68d4 100644 --- a/scrapy/newitem/loader/expanders.py +++ b/scrapy/newitem/loader/expanders.py @@ -1,5 +1,7 @@ """ -This module provides some commonly used Expanders +This module provides some commonly used Expanders. + +See documentation in docs/topics/newitem-loader.rst """ from scrapy.utils.misc import arg_to_iter @@ -7,19 +9,6 @@ from scrapy.utils.python import get_func_args from scrapy.utils.datatypes import MergeDict class TreeExpander(object): - """An expander which applies the given list of functions consecutively to - each value returned by the previous function. - - The algorithm consists in an ordered list of functions, each of which - receives one value and can return zero, one or more values (as a list or - iterable). If a function returns more than one value, the next function in - the list will be called with each of those values, potentially returning - more values and thus expanding the execution into different branches, which - is why this expander is called Tree Expander. - - The expander functions can optionally receive a ``loader_args`` argument, - which will contain the current active loader arguments. - """ def __init__(self, *functions, **default_loader_args): self.default_loader_args = default_loader_args @@ -51,9 +40,6 @@ class TreeExpander(object): class IdentityExpander(object): - """An expander which returns the original values unchanged. It doesn't - support any constructor arguments. - """ def __call__(self, values, loader_args): return arg_to_iter(values) diff --git a/scrapy/newitem/loader/reducers.py b/scrapy/newitem/loader/reducers.py index 76ba754b4..8f9a3cc9b 100644 --- a/scrapy/newitem/loader/reducers.py +++ b/scrapy/newitem/loader/reducers.py @@ -1,9 +1,10 @@ """ -This module provides some commonly used Reducers +This module provides some commonly used Reducers. + +See documentation in docs/topics/newitem-loader.rst """ class TakeFirst(object): - """Return the first non-null value from the list to reduce""" def __call__(self, values): for value in values: @@ -12,19 +13,12 @@ class TakeFirst(object): class Identity(object): - """Return the list to reduce untouched""" def __call__(self, values): return values -class JoinStrings(object): - """Return a string with the contents of the list to reduce joined with the - separator given in the constructor, which defaults to u' '. - - When using the default separator, this reducer is equivalent to the - function: u' '.join - """ +class Join(object): def __init__(self, separator=u' '): self.separator = separator diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py index 9dd949e8d..de0b1fb67 100644 --- a/scrapy/tests/test_itemloader.py +++ b/scrapy/tests/test_itemloader.py @@ -2,7 +2,7 @@ import unittest from scrapy.newitem.loader import ItemLoader from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander -from scrapy.newitem.loader.reducers import JoinStrings, Identity +from scrapy.newitem.loader.reducers import Join, Identity from scrapy.newitem import Item, Field # test items @@ -161,14 +161,14 @@ class ItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_reduced_value('name'), u'Mar') class TakeFirstItemLoader(TestItemLoader): - name_red = JoinStrings() + name_red = Join() il = TakeFirstItemLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') class TakeFirstItemLoader(TestItemLoader): - name_red = JoinStrings("
") + name_red = Join("
") il = TakeFirstItemLoader() il.add_value('name', [u'mar', u'ta'])