diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst index ae8a10baf..5635d7c04 100644 --- a/docs/experimental/loaders.rst +++ b/docs/experimental/loaders.rst @@ -115,8 +115,8 @@ positional (required) argument. The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output -processors to populate the fields, so output processors should expect iterables as -input. +processors to populate the fields, so output processors should expect iterables +as input. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. @@ -129,16 +129,16 @@ Item Loaders are declared like Items, by using a class definition syntax. Here is an example:: from scrapy.contrib.loader import ItemLoader - from scrapy.contrib.loader.processor import TakeFirst, ApplyConcat, Join + from scrapy.contrib.loader.processor import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): default_input_processor = TakeFirst() - name_in = ApplyConcat(unicode.title) + name_in = MapCompose(unicode.title) name_out = Join() - price_in = ApplyConcat(unicode.strip) + price_in = MapCompose(unicode.strip) price_out = TakeFirst() # ... @@ -161,19 +161,19 @@ output processors to use: in the :ref:`Item Field ` metadata. Here is an example:: from scrapy.newitem import Item, Field - from scrapy.contrib.loader.processor import ApplyConcat, Join, TakeFirst + from scrapy.contrib.loader.processor import MapCompose, Join, TakeFirst from scrapy.utils.markup import remove_entities from myproject.utils import filter_prices class Product(Item): name = Field( - input_processor=ApplyConcat(remove_entities), + input_processor=MapCompose(remove_entities), output_processor=Join(), ) price = Field( default=0, - input_processor=ApplyConcat(remove_entities, filter_prices), + input_processor=MapCompose(remove_entities, filter_prices), output_processor=TakeFirst(), ) @@ -224,11 +224,11 @@ There are several ways to modify Item Loader context values: p = ItemLoader(product, unit='cm') 2. On Item Loader declaration, for those input/output processors that support - instatiating them with a Item Loader context. :class:`ApplyConcat` is one of + instatiating them with a Item Loader context. :class:`MapCompose` is one of them:: class ProductLoader(ItemLoader): - length_out = ApplyConcat(parse_length, unit='cm') + length_out = MapCompose(parse_length, unit='cm') ItemLoader objects @@ -395,25 +395,25 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.contrib.loader.processor import ApplyConcat + from scrapy.contrib.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): return x.strip('-') class SiteSpecificLoader(ProductLoader): - name_in = ApplyConcat(ProductLoader.name_in, strip_dashes) + name_in = MapCompose(ProductLoader.name_in, strip_dashes) Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.contrib.loader.processor import ApplyConcat + from scrapy.contrib.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata class XmlProductLoader(ProductLoader): - name_in = ApplyConcat(remove_cdata, ProductLoader.name_in) + name_in = MapCompose(remove_cdata, ProductLoader.name_in) And that's how you typically extend input processors. @@ -438,7 +438,7 @@ Available built-in processors Even though you can use any callable function as input and output processors, Scrapy provides some commonly used processors, which are described below. Some -of them, like the :class:`ApplyConcat` (which is typically used as input +of them, like the :class:`MapCompose` (which is typically used as input processor) composes the output of several functions executed in order, to produce the final parsed value. @@ -496,9 +496,16 @@ Here is a list of all built-in processors: function, and so on, until the last function returns the output value of this processor. + Example:: + + >>> from scrapy.contrib.loader.processor import Compose + >>> proc = Compose(lambda v: v[0], str.upper) + >>> proc(['hello', 'world']) + 'HELLO' + Each function can optionally receive a ``loader_context`` parameter. For those which does this processor will pass the currently active :ref:`Loader - context ` through that parameter. + context ` through that parameter. The keyword arguments passed in the constructor are used as the default Loader context values passed to each function call. However, the final @@ -506,28 +513,44 @@ Here is a list of all built-in processors: active Loader context accessible through the :meth:`ItemLoader.context` attribute. -.. class:: ApplyConcat(\*functions, \**default_loader_context) +.. class:: MapCompose(\*functions, \**default_loader_context) - A processor which applies the given functions consecutively, in order, - concatenating their results before next function call. So each function - returns a list of values (though it could return ``None`` or a signle value - too) and the next function is called once for each of those values, - receiving one of those values as input each time. The output of each - function call (for each input value) is concatenated and each values of the - concatenation is used to call the next function, and the process repeats - until there are no functions left. - - As with the Compose processor, functions can receive Loader contexts, and - constructor keyword arguments are used as default context values. See - :class:`Compose` processor for more info. + A processor which is constructed from the composition of the given + functions, similar to the :class:`Compose` processor. The difference with + this processor is the way internal results are passed among functions, + which is as follows: - Example:: + The input value of this processor is *iterated* and each element is passed + to the first function, and the result of that function (for each element) + is concatenated to construct a new iterable, which is then passed to the + second function, and so on, until the last function is applied for each + value of the list of values collected so far. The output values of the last + function are concatenated together to produce the output of this processor. + + Each particular function can return a value or a list of values, which is + flattened with the list of values returned by the same function applied to + the other input values. The functions can also return ``None`` in which + case the output of that function is ignored for further processing over the + chain. + + This processor provides a convenient way to compose functions that only + work with single values (instead of iterables). For this reason the + :class:`MapCompose` processor is typically used as input processor, since + data is often extracted using the + :meth:`~scrapy.xpath.XPathSelector.extract` method of :ref:`selectors + `, which returns a list of unicode strings. + + The example below should clarify how it works:: >>> def filter_world(x): ... return None if x == 'world' else x ... - >>> from scrapy.contrib.loader.processor import ApplyConcat - >>> proc = ApplyConcat(filter_world, str.upper) - >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] + >>> from scrapy.contrib.loader.processor import MapCompose + >>> proc = MapCompose(filter_world, unicode.upper) + >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) + [u'HELLO, u'THIS', u'IS', u'SCRAPY'] + + As with the Compose processor, functions can receive Loader contexts, and + constructor keyword arguments are used as default context values. See + :class:`Compose` processor for more info. diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py index 725f6c6b8..26806aeb5 100644 --- a/scrapy/contrib/loader/processor.py +++ b/scrapy/contrib/loader/processor.py @@ -8,7 +8,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.datatypes import MergeDict from .common import wrap_loader_context -class ApplyConcat(object): +class MapCompose(object): def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -26,7 +26,7 @@ class ApplyConcat(object): for v in values: next_values += arg_to_iter(func(v)) values = next_values - return list(values) + return values class Compose(object): diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 014c5c057..7dbdf5806 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.loader import ItemLoader, XPathItemLoader -from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity, Compose +from scrapy.contrib.loader.processor import Join, Identity, Compose, MapCompose from scrapy.newitem import Item, Field from scrapy.xpath import HtmlXPathSelector from scrapy.http import HtmlResponse @@ -21,10 +21,10 @@ class NameItemLoader(ItemLoader): default_item_class = TestItem class TestItemLoader(NameItemLoader): - name_in = ApplyConcat(lambda v: v.title()) + name_in = MapCompose(lambda v: v.title()) class DefaultedItemLoader(NameItemLoader): - default_input_processor = ApplyConcat(lambda v: v[:-1]) + default_input_processor = MapCompose(lambda v: v[:-1]) # test processors @@ -73,13 +73,13 @@ class ItemLoaderTest(unittest.TestCase): def filter_world(x): return None if x == 'world' else x - proc = ApplyConcat(filter_world, str.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), ['HELLO', 'THIS', 'IS', 'SCRAPY']) def test_map_concat_filter_multiple_functions(self): class TestItemLoader(NameItemLoader): - name_in = ApplyConcat(lambda v: v.title(), lambda v: v[:-1]) + name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) ip = TestItemLoader() ip.add_value('name', u'marta') @@ -102,7 +102,7 @@ class ItemLoaderTest(unittest.TestCase): def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): - url_in = ApplyConcat(lambda v: v.lower()) + url_in = MapCompose(lambda v: v.lower()) ip = ChildItemLoader() ip.add_value('url', u'HTTP://scrapy.ORG') @@ -111,8 +111,8 @@ class ItemLoaderTest(unittest.TestCase): self.assertEqual(ip.get_output_value('name'), [u'Marta']) class ChildChildItemLoader(ChildItemLoader): - url_in = ApplyConcat(lambda v: v.upper()) - summary_in = ApplyConcat(lambda v: v) + url_in = MapCompose(lambda v: v.upper()) + summary_in = MapCompose(lambda v: v) ip = ChildChildItemLoader() ip.add_value('url', u'http://scrapy.org') @@ -122,7 +122,7 @@ class ItemLoaderTest(unittest.TestCase): def test_empty_map_concat(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = ApplyConcat() + name_in = MapCompose() ip = IdentityDefaultedItemLoader() ip.add_value('name', u'marta') @@ -138,7 +138,7 @@ class ItemLoaderTest(unittest.TestCase): def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): - name_in = ApplyConcat(TestItemLoader.name_in, unicode.swapcase) + name_in = MapCompose(TestItemLoader.name_in, unicode.swapcase) ip = ChildItemLoader() ip.add_value('name', u'marta') @@ -146,7 +146,7 @@ class ItemLoaderTest(unittest.TestCase): def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = ApplyConcat(DefaultedItemLoader.default_input_processor, unicode.swapcase) + name_in = MapCompose(DefaultedItemLoader.default_input_processor, unicode.swapcase) ip = ChildDefaultedItemLoader() ip.add_value('name', u'marta') @@ -197,7 +197,7 @@ class ItemLoaderTest(unittest.TestCase): def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = ApplyConcat(processor_with_args, key=u'val') + url_in = MapCompose(processor_with_args, key=u'val') ip = ChildItemLoader() ip.add_value('url', u'text') @@ -207,7 +207,7 @@ class ItemLoaderTest(unittest.TestCase): def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): - url_in = ApplyConcat(processor_with_args) + url_in = MapCompose(processor_with_args) ip = ChildItemLoader(key=u'val') ip.add_value('url', u'text') @@ -217,7 +217,7 @@ class ItemLoaderTest(unittest.TestCase): def test_loader_context_on_assign(self): class ChildItemLoader(TestItemLoader): - url_in = ApplyConcat(processor_with_args) + url_in = MapCompose(processor_with_args) ip = ChildItemLoader() ip.context['key'] = u'val' @@ -231,7 +231,7 @@ class ItemLoaderTest(unittest.TestCase): return loader_context['item']['name'] class ChildItemLoader(TestItemLoader): - url_in = ApplyConcat(processor) + url_in = MapCompose(processor) it = TestItem(name='marta') ip = ChildItemLoader(item=it) @@ -257,7 +257,7 @@ class ItemLoaderTest(unittest.TestCase): class TestXPathItemLoader(XPathItemLoader): default_item_class = TestItem - name_in = ApplyConcat(lambda v: v.title()) + name_in = MapCompose(lambda v: v.title()) class XPathItemLoaderTest(unittest.TestCase):