RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

renamed ItemLoader class to Loader

This commit is contained in:
Pablo Hoffman 2009-08-07 14:39:30 -03:00
parent e585c6cac4
commit db90e26a8b
3 changed files with 63 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
docs/experimental/loaders.rst
View File

@ -27,7 +27,7 @@ Using Loaders to populate items
To use a Loader, you must first instantiate it. You can either instantiate it
with an Item object or without one, in which case an Item is automatically
instantiated in the Loader constructor using the Item class specified in the
:attr:`ItemLoader.default_item_class` attribute.
:attr:`Loader.default_item_class` attribute.
Then, you start adding values to the Loader, typically collecting them using
:ref:`Selectors <topics-selectors>`. You can add more than one value to the
@ -43,7 +43,7 @@ Here is a typical Loader usage in a :ref:`Spider <topics-spiders>` using the
def parse(self, response):
x = HtmlXPathSelector(response)
l = ItemLoader(item=Product())
l = Loader(item=Product())
l.add_value('name', x.x('//div[@class="product_name"]').extract())
l.add_value('name', x.x('//div[@class="product_title"]').extract())
l.add_value('price', x.x('//p[@id="price"]').extract())
@ -64,7 +64,7 @@ Afterwards, similar calls are used for ``price`` and ``stock`` fields, and
finally the ``last_update`` field is populated directly with a literal value
(``today``).
Finally, when all data is collected, the :meth:`ItemLoader.get_item` method is
Finally, when all data is collected, the :meth:`Loader.get_item` method is
called which actually populates and returns the item populated with the data
previously extracted with the ``add_value`` calls.
@ -73,9 +73,9 @@ Expanders and Reducers
A Loader is composed of one expander and reducer for each item field. The
Expander processes the extracted data as soon as it's received through the
:meth:`ItemLoader.add_value` method, and the result of the expander is collected
:meth:`Loader.add_value` method, and the result of the expander is collected
and kept inside the Loader. After collecting all data, the
:meth:`ItemLoader.get_item` method is called to actually populate and get the Item.
:meth:`Loader.get_item` method is called to actually populate and get the Item.
That's when the Reducers are called with the data previously collected (using
the Expanders) and the output of the Reducers are the actual values that get
assigned to the item.
@ -83,7 +83,7 @@ assigned to the item.
Let's see an example to illustrate how Expanders and Reducers are called, for a
particular field (the same applies for any other field)::
l = ItemLoader(Product())
l = Loader(Product())
l.add_value('name', x.x(xpath1).extract()) # (1)
l.add_value('name', x.x(xpath2).extract()) # (2)
return l.get_item() # (3)
@ -112,11 +112,11 @@ Declaring Loaders
Loaders are declared like Items, by using a class definition syntax. Here is an
example::
from scrapy.newitem.loader import ItemLoader
from scrapy.newitem.loader import Loader
from scrapy.newitem.loader.expanders import TreeExpander
from scrapy.newitem.loader.reducers import Join, TakeFirst
class ProductLoader(ItemLoader):
class ProductLoader(Loader):
default_expander = TakeFirst()
@ -130,7 +130,7 @@ example::
As you can see, expanders are declared using the ``_exp`` suffix while reducers
are declared using the ``_red`` suffix. And you can also declare a default
expander using the :attr:`ItemLoader.default_expander` attribute.
expander using the :attr:`Loader.default_expander` attribute.
.. _topics-loader-args:
@ -156,21 +156,21 @@ There are seveal ways to pass loader arguments:
1. Passing arguments on Loader declaration::
class ProductLoader(ItemLoader):
class ProductLoader(Loader):
length_exp = TreeExpander(parse_length, unit='cm')
2. Passing arguments on Loader instantiation::
l = ItemLoader(product, unit='cm')
l = Loader(product, unit='cm')
3. Passing arguments on Loader usage::
l.add_value('length', x.x('//div').extract(), unit='cm')
ItemLoader objects
==================
Loader objects
==============
.. class:: ItemLoader([item], \**loader_args)
.. class:: Loader([item], \**loader_args)
Return a new Item Loader for populating the given Item. If no item is
given, one is instantiated using the class in :attr:`default_item_class`.

2
scrapy/newitem/loader/__init__.py
View File

@ -5,7 +5,7 @@ from scrapy.newitem import Item
from scrapy.newitem.loader.reducers import TakeFirst
from scrapy.newitem.loader.expanders import IdentityExpander
class ItemLoader(object):
class Loader(object):
default_item_class = Item
default_expander = IdentityExpander()

96
scrapy/tests/test_itemloader.py
View File

@ -1,6 +1,6 @@
import unittest
from scrapy.newitem.loader import ItemLoader
from scrapy.newitem.loader import Loader
from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander
from scrapy.newitem.loader.reducers import Join, Identity
from scrapy.newitem import Item, Field
@ -16,13 +16,13 @@ class TestItem(NameItem):
# test loaders
class NameItemLoader(ItemLoader):
class NameLoader(Loader):
default_item_class = TestItem
class TestItemLoader(NameItemLoader):
class TestLoader(NameLoader):
name_exp = TreeExpander(lambda v: v.title())
class DefaultedItemLoader(NameItemLoader):
class DefaultedLoader(NameLoader):
default_expander = TreeExpander(lambda v: v[:-1])
# test expanders
@ -32,12 +32,12 @@ def expander_func_with_args(value, other=None, loader_args=None):
return loader_args['key']
return value
class ItemLoaderTest(unittest.TestCase):
class Loader(unittest.TestCase):
def test_get_item_using_default_loader(self):
i = TestItem()
i['summary'] = u'lala'
il = ItemLoader(item=i)
il = Loader(item=i)
il.add_value('name', u'marta')
item = il.get_item()
assert item is i
@ -45,13 +45,13 @@ class ItemLoaderTest(unittest.TestCase):
self.assertEqual(item['name'], u'marta')
def test_get_item_using_custom_loader(self):
il = TestItemLoader()
il = TestLoader()
il.add_value('name', u'marta')
item = il.get_item()
self.assertEqual(item['name'], u'Marta')
def test_add_value(self):
il = TestItemLoader()
il = TestLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
self.assertEqual(il.get_reduced_value('name'), u'Marta')
@ -60,7 +60,7 @@ class ItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_reduced_value('name'), u'Marta')
def test_replace_value(self):
il = TestItemLoader()
il = TestLoader()
il.replace_value('name', u'marta')
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
self.assertEqual(il.get_reduced_value('name'), u'Marta')
@ -69,144 +69,144 @@ class ItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_reduced_value('name'), u'Pepe')
def test_tree_expander_multiple_functions(self):
class TestItemLoader(NameItemLoader):
class TestLoader(NameLoader):
name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
il = TestLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'Mart')
item = il.get_item()
self.assertEqual(item['name'], u'Mart')
def test_default_expander(self):
il = DefaultedItemLoader()
il = DefaultedLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'mart')
def test_inherited_default_expander(self):
class InheritDefaultedItemLoader(DefaultedItemLoader):
class InheritDefaultedLoader(DefaultedLoader):
pass
il = InheritDefaultedItemLoader()
il = InheritDefaultedLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'mart')
def test_expander_inheritance(self):
class ChildItemLoader(TestItemLoader):
class ChildLoader(TestLoader):
url_exp = TreeExpander(lambda v: v.lower())
il = ChildItemLoader()
il = ChildLoader()
il.add_value('url', u'HTTP://scrapy.ORG')
self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org')
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'Marta')
class ChildChildItemLoader(ChildItemLoader):
class ChildChildLoader(ChildLoader):
url_exp = TreeExpander(lambda v: v.upper())
summary_exp = TreeExpander(lambda v: v)
il = ChildChildItemLoader()
il = ChildChildLoader()
il.add_value('url', u'http://scrapy.org')
self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG')
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'Marta')
def test_empty_tree_expander(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
class IdentityDefaultedLoader(DefaultedLoader):
name_exp = TreeExpander()
il = IdentityDefaultedItemLoader()
il = IdentityDefaultedLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'marta')
def test_identity_expander(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
class IdentityDefaultedLoader(DefaultedLoader):
name_exp = IdentityExpander()
il = IdentityDefaultedItemLoader()
il = IdentityDefaultedLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'marta')
def test_extend_custom_expanders(self):
class ChildItemLoader(TestItemLoader):
name_exp = TreeExpander(TestItemLoader.name_exp, unicode.swapcase)
class ChildLoader(TestLoader):
name_exp = TreeExpander(TestLoader.name_exp, unicode.swapcase)
il = ChildItemLoader()
il = ChildLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'mARTA')
def test_extend_default_expanders(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
name_exp = TreeExpander(DefaultedItemLoader.default_expander, unicode.swapcase)
class ChildDefaultedLoader(DefaultedLoader):
name_exp = TreeExpander(DefaultedLoader.default_expander, unicode.swapcase)
il = ChildDefaultedItemLoader()
il = ChildDefaultedLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_reduced_value('name'), u'MART')
def test_reducer_using_function(self):
il = TestItemLoader()
il = TestLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar')
class TakeFirstItemLoader(TestItemLoader):
class TakeFirstLoader(TestLoader):
name_red = u" ".join
il = TakeFirstItemLoader()
il = TakeFirstLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar Ta')
def test_reducer_using_classes(self):
il = TestItemLoader()
il = TestLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar')
class TakeFirstItemLoader(TestItemLoader):
class TakeFirstLoader(TestLoader):
name_red = Join()
il = TakeFirstItemLoader()
il = TakeFirstLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar Ta')
class TakeFirstItemLoader(TestItemLoader):
class TakeFirstLoader(TestLoader):
name_red = Join("<br>")
il = TakeFirstItemLoader()
il = TakeFirstLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar<br>Ta')
def test_default_reducer(self):
il = TestItemLoader()
il = TestLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), u'Mar')
class LalaItemLoader(TestItemLoader):
class LalaLoader(TestLoader):
default_reducer = Identity()
il = LalaItemLoader()
il = LalaLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta'])
def test_expander_args_on_declaration(self):
class ChildItemLoader(TestItemLoader):
class ChildLoader(TestLoader):
url_exp = TreeExpander(expander_func_with_args, key=u'val')
il = ChildItemLoader()
il = ChildLoader()
il.add_value('url', u'text', key=u'val')
self.assertEqual(il.get_reduced_value('url'), 'val')
def test_expander_args_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
class ChildLoader(TestLoader):
url_exp = TreeExpander(expander_func_with_args)
il = ChildItemLoader(key=u'val')
il = ChildLoader(key=u'val')
il.add_value('url', u'text')
self.assertEqual(il.get_reduced_value('url'), 'val')
def test_expander_args_on_assign(self):
class ChildItemLoader(TestItemLoader):
class ChildLoadeLoader(TestLoader):
url_exp = TreeExpander(expander_func_with_args)
il = ChildItemLoader()
il = ChildLoader()
il.add_value('url', u'text', key=u'val')
self.assertEqual(il.get_reduced_value('url'), 'val')
@ -214,16 +214,16 @@ class ItemLoaderTest(unittest.TestCase):
def exp_func(value, loader_args):
return loader_args['item']['name']
class ChildItemLoader(TestItemLoader):
class ChildLoader(TestLoader):
url_exp = TreeExpander(exp_func)
it = TestItem(name='marta')
il = ChildItemLoader(item=it)
il = ChildLoader(item=it)
il.add_value('url', u'text', key=u'val')
self.assertEqual(il.get_reduced_value('url'), 'marta')
def test_add_value_on_unknown_field(self):
il = TestItemLoader()
il = TestLoader()
self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
if __name__ == "__main__":