diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst index c66b4448b..a4350d89c 100644 --- a/docs/experimental/loaders.rst +++ b/docs/experimental/loaders.rst @@ -27,7 +27,7 @@ Using Loaders to populate items To use a Loader, you must first instantiate it. You can either instantiate it with an Item object or without one, in which case an Item is automatically instantiated in the Loader constructor using the Item class specified in the -:attr:`ItemLoader.default_item_class` attribute. +:attr:`Loader.default_item_class` attribute. Then, you start adding values to the Loader, typically collecting them using :ref:`Selectors `. You can add more than one value to the @@ -43,7 +43,7 @@ Here is a typical Loader usage in a :ref:`Spider ` using the def parse(self, response): x = HtmlXPathSelector(response) - l = ItemLoader(item=Product()) + l = Loader(item=Product()) l.add_value('name', x.x('//div[@class="product_name"]').extract()) l.add_value('name', x.x('//div[@class="product_title"]').extract()) l.add_value('price', x.x('//p[@id="price"]').extract()) @@ -64,7 +64,7 @@ Afterwards, similar calls are used for ``price`` and ``stock`` fields, and finally the ``last_update`` field is populated directly with a literal value (``today``). -Finally, when all data is collected, the :meth:`ItemLoader.get_item` method is +Finally, when all data is collected, the :meth:`Loader.get_item` method is called which actually populates and returns the item populated with the data previously extracted with the ``add_value`` calls. @@ -73,9 +73,9 @@ Expanders and Reducers A Loader is composed of one expander and reducer for each item field. The Expander processes the extracted data as soon as it's received through the -:meth:`ItemLoader.add_value` method, and the result of the expander is collected +:meth:`Loader.add_value` method, and the result of the expander is collected and kept inside the Loader. After collecting all data, the -:meth:`ItemLoader.get_item` method is called to actually populate and get the Item. +:meth:`Loader.get_item` method is called to actually populate and get the Item. That's when the Reducers are called with the data previously collected (using the Expanders) and the output of the Reducers are the actual values that get assigned to the item. @@ -83,7 +83,7 @@ assigned to the item. Let's see an example to illustrate how Expanders and Reducers are called, for a particular field (the same applies for any other field):: - l = ItemLoader(Product()) + l = Loader(Product()) l.add_value('name', x.x(xpath1).extract()) # (1) l.add_value('name', x.x(xpath2).extract()) # (2) return l.get_item() # (3) @@ -112,11 +112,11 @@ Declaring Loaders Loaders are declared like Items, by using a class definition syntax. Here is an example:: - from scrapy.newitem.loader import ItemLoader + from scrapy.newitem.loader import Loader from scrapy.newitem.loader.expanders import TreeExpander from scrapy.newitem.loader.reducers import Join, TakeFirst - class ProductLoader(ItemLoader): + class ProductLoader(Loader): default_expander = TakeFirst() @@ -130,7 +130,7 @@ example:: As you can see, expanders are declared using the ``_exp`` suffix while reducers are declared using the ``_red`` suffix. And you can also declare a default -expander using the :attr:`ItemLoader.default_expander` attribute. +expander using the :attr:`Loader.default_expander` attribute. .. _topics-loader-args: @@ -156,21 +156,21 @@ There are seveal ways to pass loader arguments: 1. Passing arguments on Loader declaration:: - class ProductLoader(ItemLoader): + class ProductLoader(Loader): length_exp = TreeExpander(parse_length, unit='cm') 2. Passing arguments on Loader instantiation:: - l = ItemLoader(product, unit='cm') + l = Loader(product, unit='cm') 3. Passing arguments on Loader usage:: l.add_value('length', x.x('//div').extract(), unit='cm') -ItemLoader objects -================== +Loader objects +============== -.. class:: ItemLoader([item], \**loader_args) +.. class:: Loader([item], \**loader_args) Return a new Item Loader for populating the given Item. If no item is given, one is instantiated using the class in :attr:`default_item_class`. diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py index 09640aabb..cf8fe36a0 100644 --- a/scrapy/newitem/loader/__init__.py +++ b/scrapy/newitem/loader/__init__.py @@ -5,7 +5,7 @@ from scrapy.newitem import Item from scrapy.newitem.loader.reducers import TakeFirst from scrapy.newitem.loader.expanders import IdentityExpander -class ItemLoader(object): +class Loader(object): default_item_class = Item default_expander = IdentityExpander() diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py index de0b1fb67..a008ce1c4 100644 --- a/scrapy/tests/test_itemloader.py +++ b/scrapy/tests/test_itemloader.py @@ -1,6 +1,6 @@ import unittest -from scrapy.newitem.loader import ItemLoader +from scrapy.newitem.loader import Loader from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander from scrapy.newitem.loader.reducers import Join, Identity from scrapy.newitem import Item, Field @@ -16,13 +16,13 @@ class TestItem(NameItem): # test loaders -class NameItemLoader(ItemLoader): +class NameLoader(Loader): default_item_class = TestItem -class TestItemLoader(NameItemLoader): +class TestLoader(NameLoader): name_exp = TreeExpander(lambda v: v.title()) -class DefaultedItemLoader(NameItemLoader): +class DefaultedLoader(NameLoader): default_expander = TreeExpander(lambda v: v[:-1]) # test expanders @@ -32,12 +32,12 @@ def expander_func_with_args(value, other=None, loader_args=None): return loader_args['key'] return value -class ItemLoaderTest(unittest.TestCase): +class Loader(unittest.TestCase): def test_get_item_using_default_loader(self): i = TestItem() i['summary'] = u'lala' - il = ItemLoader(item=i) + il = Loader(item=i) il.add_value('name', u'marta') item = il.get_item() assert item is i @@ -45,13 +45,13 @@ class ItemLoaderTest(unittest.TestCase): self.assertEqual(item['name'], u'marta') def test_get_item_using_custom_loader(self): - il = TestItemLoader() + il = TestLoader() il.add_value('name', u'marta') item = il.get_item() self.assertEqual(item['name'], u'Marta') def test_add_value(self): - il = TestItemLoader() + il = TestLoader() il.add_value('name', u'marta') self.assertEqual(il.get_expanded_value('name'), [u'Marta']) self.assertEqual(il.get_reduced_value('name'), u'Marta') @@ -60,7 +60,7 @@ class ItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_reduced_value('name'), u'Marta') def test_replace_value(self): - il = TestItemLoader() + il = TestLoader() il.replace_value('name', u'marta') self.assertEqual(il.get_expanded_value('name'), [u'Marta']) self.assertEqual(il.get_reduced_value('name'), u'Marta') @@ -69,144 +69,144 @@ class ItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_reduced_value('name'), u'Pepe') def test_tree_expander_multiple_functions(self): - class TestItemLoader(NameItemLoader): + class TestLoader(NameLoader): name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1]) - il = TestItemLoader() + il = TestLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'Mart') item = il.get_item() self.assertEqual(item['name'], u'Mart') def test_default_expander(self): - il = DefaultedItemLoader() + il = DefaultedLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'mart') def test_inherited_default_expander(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): + class InheritDefaultedLoader(DefaultedLoader): pass - il = InheritDefaultedItemLoader() + il = InheritDefaultedLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'mart') def test_expander_inheritance(self): - class ChildItemLoader(TestItemLoader): + class ChildLoader(TestLoader): url_exp = TreeExpander(lambda v: v.lower()) - il = ChildItemLoader() + il = ChildLoader() il.add_value('url', u'HTTP://scrapy.ORG') self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org') il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'Marta') - class ChildChildItemLoader(ChildItemLoader): + class ChildChildLoader(ChildLoader): url_exp = TreeExpander(lambda v: v.upper()) summary_exp = TreeExpander(lambda v: v) - il = ChildChildItemLoader() + il = ChildChildLoader() il.add_value('url', u'http://scrapy.org') self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG') il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'Marta') def test_empty_tree_expander(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): + class IdentityDefaultedLoader(DefaultedLoader): name_exp = TreeExpander() - il = IdentityDefaultedItemLoader() + il = IdentityDefaultedLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'marta') def test_identity_expander(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): + class IdentityDefaultedLoader(DefaultedLoader): name_exp = IdentityExpander() - il = IdentityDefaultedItemLoader() + il = IdentityDefaultedLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'marta') def test_extend_custom_expanders(self): - class ChildItemLoader(TestItemLoader): - name_exp = TreeExpander(TestItemLoader.name_exp, unicode.swapcase) + class ChildLoader(TestLoader): + name_exp = TreeExpander(TestLoader.name_exp, unicode.swapcase) - il = ChildItemLoader() + il = ChildLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'mARTA') def test_extend_default_expanders(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_exp = TreeExpander(DefaultedItemLoader.default_expander, unicode.swapcase) + class ChildDefaultedLoader(DefaultedLoader): + name_exp = TreeExpander(DefaultedLoader.default_expander, unicode.swapcase) - il = ChildDefaultedItemLoader() + il = ChildDefaultedLoader() il.add_value('name', u'marta') self.assertEqual(il.get_reduced_value('name'), u'MART') def test_reducer_using_function(self): - il = TestItemLoader() + il = TestLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar') - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstLoader(TestLoader): name_red = u" ".join - il = TakeFirstItemLoader() + il = TakeFirstLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') def test_reducer_using_classes(self): - il = TestItemLoader() + il = TestLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar') - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstLoader(TestLoader): name_red = Join() - il = TakeFirstItemLoader() + il = TakeFirstLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstLoader(TestLoader): name_red = Join("
") - il = TakeFirstItemLoader() + il = TakeFirstLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar
Ta') def test_default_reducer(self): - il = TestItemLoader() + il = TestLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), u'Mar') - class LalaItemLoader(TestItemLoader): + class LalaLoader(TestLoader): default_reducer = Identity() - il = LalaItemLoader() + il = LalaLoader() il.add_value('name', [u'mar', u'ta']) self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta']) def test_expander_args_on_declaration(self): - class ChildItemLoader(TestItemLoader): + class ChildLoader(TestLoader): url_exp = TreeExpander(expander_func_with_args, key=u'val') - il = ChildItemLoader() + il = ChildLoader() il.add_value('url', u'text', key=u'val') self.assertEqual(il.get_reduced_value('url'), 'val') def test_expander_args_on_instantiation(self): - class ChildItemLoader(TestItemLoader): + class ChildLoader(TestLoader): url_exp = TreeExpander(expander_func_with_args) - il = ChildItemLoader(key=u'val') + il = ChildLoader(key=u'val') il.add_value('url', u'text') self.assertEqual(il.get_reduced_value('url'), 'val') def test_expander_args_on_assign(self): - class ChildItemLoader(TestItemLoader): + class ChildLoadeLoader(TestLoader): url_exp = TreeExpander(expander_func_with_args) - il = ChildItemLoader() + il = ChildLoader() il.add_value('url', u'text', key=u'val') self.assertEqual(il.get_reduced_value('url'), 'val') @@ -214,16 +214,16 @@ class ItemLoaderTest(unittest.TestCase): def exp_func(value, loader_args): return loader_args['item']['name'] - class ChildItemLoader(TestItemLoader): + class ChildLoader(TestLoader): url_exp = TreeExpander(exp_func) it = TestItem(name='marta') - il = ChildItemLoader(item=it) + il = ChildLoader(item=it) il.add_value('url', u'text', key=u'val') self.assertEqual(il.get_reduced_value('url'), 'marta') def test_add_value_on_unknown_field(self): - il = TestItemLoader() + il = TestLoader() self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) if __name__ == "__main__":