mirror of https://github.com/scrapy/scrapy.git
add support for a nested loaders
This commit is contained in:
Daniel Collins
Daniel Collins
parent
70fa3ae9a5
commit
1f95af3c07
|
|
@ -24,16 +24,46 @@ class ItemLoader(object):
|
|||
default_output_processor = Identity()
|
||||
default_selector_class = Selector
|
||||
|
||||
def __init__(self, item=None, selector=None, response=None, **context):
|
||||
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
|
||||
if selector is None and response is not None:
|
||||
selector = self.default_selector_class(response)
|
||||
self.selector = selector
|
||||
context.update(selector=selector, response=response)
|
||||
if item is None:
|
||||
item = self.default_item_class()
|
||||
self.item = context['item'] = item
|
||||
self.context = context
|
||||
self._values = defaultdict(list)
|
||||
self.parent = parent
|
||||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
|
||||
@property
|
||||
def _values(self):
|
||||
if self.parent is not None:
|
||||
return self.parent._values
|
||||
else:
|
||||
return self._local_values
|
||||
|
||||
@property
|
||||
def item(self):
|
||||
if self.parent is not None:
|
||||
return self.parent.item
|
||||
else:
|
||||
return self._local_item
|
||||
|
||||
def nested_loader(self, xpath=None, css=None):
|
||||
if xpath is not None and css is not None:
|
||||
raise ValueError("Cannot nest a loader with both a xpath selector and a css selector")
|
||||
|
||||
if xpath is not None:
|
||||
selector = self.selector.xpath(xpath)
|
||||
|
||||
if css is not None:
|
||||
selector = self.selector.css(css)
|
||||
|
||||
subloader = self.__class__(
|
||||
item=self.item, selector=selector, parent=self
|
||||
)
|
||||
return subloader
|
||||
|
||||
def add_value(self, field_name, value, *processors, **kw):
|
||||
value = self.get_value(value, *processors, **kw)
|
||||
|
|
@ -84,6 +114,10 @@ class ItemLoader(object):
|
|||
value = self.get_output_value(field_name)
|
||||
if value is not None:
|
||||
item[field_name] = value
|
||||
|
||||
# for loader in self._subloaders:
|
||||
# loader.load_item()
|
||||
|
||||
return item
|
||||
|
||||
def get_output_value(self, field_name):
|
||||
|
|
@ -168,5 +202,4 @@ class ItemLoader(object):
|
|||
csss = arg_to_iter(csss)
|
||||
return flatten([self.selector.css(css).extract() for css in csss])
|
||||
|
||||
|
||||
XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
|
||||
|
|
|
|||
|
|
@ -19,11 +19,24 @@ class TestItem(NameItem):
|
|||
summary = Field()
|
||||
|
||||
|
||||
class TestNestedItem(Item):
|
||||
name = Field()
|
||||
name_div = Field()
|
||||
name_value = Field()
|
||||
|
||||
url = Field()
|
||||
image = Field()
|
||||
|
||||
|
||||
# test item loaders
|
||||
class NameItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
|
||||
|
||||
class NestedItemLoader(ItemLoader):
|
||||
default_item_class = TestNestedItem
|
||||
|
||||
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_in = MapCompose(lambda v: v.title())
|
||||
|
||||
|
|
@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
|
||||
|
||||
|
||||
class SubselectorLoaderTest(unittest.TestCase):
|
||||
response = HtmlResponse(url="", encoding='utf-8', body=b"""
|
||||
<html>
|
||||
<body>
|
||||
<header>
|
||||
<div id="id">marta</div>
|
||||
<p>paragraph</p>
|
||||
</header>
|
||||
<footer class="footer">
|
||||
<a href="http://www.scrapy.org">homepage</a>
|
||||
<img src="/images/logo.png" width="244" height="65" alt="Scrapy">
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
def test_nested_xpath(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl = l.nested_loader(xpath="//header")
|
||||
nl.add_xpath('name', 'div/text()')
|
||||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
|
||||
|
||||
def test_nested_css(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl = l.nested_loader(css="header")
|
||||
nl.add_xpath('name', 'div/text()')
|
||||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
|
||||
|
||||
def test_nested_replace(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl1 = l.nested_loader(xpath='//footer')
|
||||
nl2 = nl1.nested_loader(xpath='a')
|
||||
|
||||
l.add_xpath('url', '//footer/a/@href')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
nl1.replace_xpath('url', 'img/@src')
|
||||
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
|
||||
nl2.replace_xpath('url', '@href')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
|
||||
def test_nested_ordering(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl1 = l.nested_loader(xpath='//footer')
|
||||
nl2 = nl1.nested_loader(xpath='a')
|
||||
|
||||
nl1.add_xpath('url', 'img/@src')
|
||||
l.add_xpath('url', '//footer/a/@href')
|
||||
nl2.add_xpath('url', 'text()')
|
||||
l.add_xpath('url', '//footer/a/@href')
|
||||
|
||||
self.assertEqual(l.get_output_value('url'), [
|
||||
u'/images/logo.png',
|
||||
u'http://www.scrapy.org',
|
||||
u'homepage',
|
||||
u'http://www.scrapy.org',
|
||||
])
|
||||
|
||||
def test_nested_load_item(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl1 = l.nested_loader(xpath='//footer')
|
||||
nl2 = nl1.nested_loader(xpath='img')
|
||||
|
||||
l.add_xpath('name', '//header/div/text()')
|
||||
nl1.add_xpath('url', 'a/@href')
|
||||
nl2.add_xpath('image', '@src')
|
||||
|
||||
item = l.load_item()
|
||||
|
||||
assert item is l.item
|
||||
assert item is nl1.item
|
||||
assert item is nl2.item
|
||||
|
||||
self.assertEqual(item['name'], [u'marta'])
|
||||
self.assertEqual(item['url'], [u'http://www.scrapy.org'])
|
||||
self.assertEqual(item['image'], [u'/images/logo.png'])
|
||||
|
||||
|
||||
class SelectJmesTestCase(unittest.TestCase):
|
||||
test_list_equals = {
|
||||
'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue