diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 84640f0b6..102ffc351 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -24,16 +24,46 @@ class ItemLoader(object): default_output_processor = Identity() default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, **context): + def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: selector = self.default_selector_class(response) self.selector = selector context.update(selector=selector, response=response) if item is None: item = self.default_item_class() - self.item = context['item'] = item self.context = context - self._values = defaultdict(list) + self.parent = parent + self._local_item = context['item'] = item + self._local_values = defaultdict(list) + + @property + def _values(self): + if self.parent is not None: + return self.parent._values + else: + return self._local_values + + @property + def item(self): + if self.parent is not None: + return self.parent.item + else: + return self._local_item + + def nested_loader(self, xpath=None, css=None): + if xpath is not None and css is not None: + raise ValueError("Cannot nest a loader with both a xpath selector and a css selector") + + if xpath is not None: + selector = self.selector.xpath(xpath) + + if css is not None: + selector = self.selector.css(css) + + subloader = self.__class__( + item=self.item, selector=selector, parent=self + ) + return subloader def add_value(self, field_name, value, *processors, **kw): value = self.get_value(value, *processors, **kw) @@ -84,6 +114,10 @@ class ItemLoader(object): value = self.get_output_value(field_name) if value is not None: item[field_name] = value + + # for loader in self._subloaders: + # loader.load_item() + return item def get_output_value(self, field_name): @@ -168,5 +202,4 @@ class ItemLoader(object): csss = arg_to_iter(csss) return flatten([self.selector.css(css).extract() for css in csss]) - XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8cf5e484a..2d34da810 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -19,11 +19,24 @@ class TestItem(NameItem): summary = Field() +class TestNestedItem(Item): + name = Field() + name_div = Field() + name_value = Field() + + url = Field() + image = Field() + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem +class NestedItemLoader(ItemLoader): + default_item_class = TestNestedItem + + class TestItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) +class SubselectorLoaderTest(unittest.TestCase): + response = HtmlResponse(url="", encoding='utf-8', body=b""" + + +
+
marta
+

paragraph

+
+ + + + """) + + def test_nested_xpath(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_loader(xpath="//header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_css(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_loader(css="header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_replace(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='a') + + l.add_xpath('url', '//footer/a/@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + nl1.replace_xpath('url', 'img/@src') + self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + nl2.replace_xpath('url', '@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + + def test_nested_ordering(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='a') + + nl1.add_xpath('url', 'img/@src') + l.add_xpath('url', '//footer/a/@href') + nl2.add_xpath('url', 'text()') + l.add_xpath('url', '//footer/a/@href') + + self.assertEqual(l.get_output_value('url'), [ + u'/images/logo.png', + u'http://www.scrapy.org', + u'homepage', + u'http://www.scrapy.org', + ]) + + def test_nested_load_item(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='img') + + l.add_xpath('name', '//header/div/text()') + nl1.add_xpath('url', 'a/@href') + nl2.add_xpath('image', '@src') + + item = l.load_item() + + assert item is l.item + assert item is nl1.item + assert item is nl2.item + + self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['url'], [u'http://www.scrapy.org']) + self.assertEqual(item['image'], [u'/images/logo.png']) + + class SelectJmesTestCase(unittest.TestCase): test_list_equals = { 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),