diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 8841711ea..b188a0b81 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -432,6 +432,22 @@ ItemLoader objects ` to get the final value to assign to each item field. + .. method:: nested_xpath(xpath) + + Create a nested loader with an xpath selector. + The supplied selector is applied relative to selector associated + with this :class:`ItemLoader`. The nested loader shares the :class:`Item` + with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, + :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + + .. method:: nested_css(css) + + Create a nested loader with a css selector. + The supplied selector is applied relative to selector associated + with this :class:`ItemLoader`. The nested loader shares the :class:`Item` + with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, + :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + .. method:: get_collected_values(field_name) Return the collected values for the given field. @@ -490,6 +506,52 @@ ItemLoader objects :attr:`default_selector_class`. This attribute is meant to be read-only. +.. _topics-loaders-nested: + +Nested Loaders +============== + +When parsing related values from a subsection of a document, it can be +useful to create nested loaders. Imagine you're extracting details from +a footer of a page that looks something like: + +Example:: + + + +Without nested loaders, you need to specify the full xpath (or css) for each value +that you wish to extract. + +Example:: + + loader = ItemLoader(item=Item()) + # load stuff not in the footer + loader.add_xpath('social', '//footer/a[@class = "social"]/@href') + loader.add_xpath('email', '//footer/a[@class = "email"]/@href') + loader.load_item() + +Instead, you can create a nested loader with the footer selector and add values +relative to the footer. The functionality is the same but you avoid repeating +the footer selector. + +Example:: + + loader = ItemLoader(item=Item()) + # load stuff not in the footer + footer_loader = loader.nested_xpath('//footer') + footer_loader.add_xpath('social', 'a[@class = "social"]/@href') + footer_loader.add_xpath('email', 'a[@class = "email"]/@href') + # no need to call footer_loader.load_item() + loader.load_item() + +You can nest loaders arbitrarilly and they work with either xpath or css selectors. +As a general guideline, use nested loaders when they make your code simpler but do +not go overboard with nesting or your parser can become difficult to read. + .. _topics-loaders-extending: Reusing and extending Item Loaders diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 6c2ff968e..e73413318 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -24,16 +24,47 @@ class ItemLoader(object): default_output_processor = Identity() default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, **context): + def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: selector = self.default_selector_class(response) self.selector = selector context.update(selector=selector, response=response) if item is None: item = self.default_item_class() - self.item = context['item'] = item self.context = context - self._values = defaultdict(list) + self.parent = parent + self._local_item = context['item'] = item + self._local_values = defaultdict(list) + + @property + def _values(self): + if self.parent is not None: + return self.parent._values + else: + return self._local_values + + @property + def item(self): + if self.parent is not None: + return self.parent.item + else: + return self._local_item + + def nested_xpath(self, xpath, **context): + selector = self.selector.xpath(xpath) + context.update(selector=selector) + subloader = self.__class__( + item=self.item, parent=self, **context + ) + return subloader + + def nested_css(self, css, **context): + selector = self.selector.css(css) + context.update(selector=selector) + subloader = self.__class__( + item=self.item, parent=self, **context + ) + return subloader def add_value(self, field_name, value, *processors, **kw): value = self.get_value(value, *processors, **kw) @@ -84,6 +115,7 @@ class ItemLoader(object): value = self.get_output_value(field_name) if value is not None: item[field_name] = value + return item def get_output_value(self, field_name): @@ -168,5 +200,4 @@ class ItemLoader(object): csss = arg_to_iter(csss) return flatten(self.selector.css(css).extract() for css in csss) - XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8cf5e484a..2693a18d9 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -19,11 +19,24 @@ class TestItem(NameItem): summary = Field() +class TestNestedItem(Item): + name = Field() + name_div = Field() + name_value = Field() + + url = Field() + image = Field() + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem +class NestedItemLoader(ItemLoader): + default_item_class = TestNestedItem + + class TestItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) +class SubselectorLoaderTest(unittest.TestCase): + response = HtmlResponse(url="", encoding='utf-8', body=b""" + + +
+
marta
+

paragraph

+
+ + + + """) + + def test_nested_xpath(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_xpath("//header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_css(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_css("header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_replace(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('a') + + l.add_xpath('url', '//footer/a/@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + nl1.replace_xpath('url', 'img/@src') + self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + nl2.replace_xpath('url', '@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + + def test_nested_ordering(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('a') + + nl1.add_xpath('url', 'img/@src') + l.add_xpath('url', '//footer/a/@href') + nl2.add_xpath('url', 'text()') + l.add_xpath('url', '//footer/a/@href') + + self.assertEqual(l.get_output_value('url'), [ + u'/images/logo.png', + u'http://www.scrapy.org', + u'homepage', + u'http://www.scrapy.org', + ]) + + def test_nested_load_item(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('img') + + l.add_xpath('name', '//header/div/text()') + nl1.add_xpath('url', 'a/@href') + nl2.add_xpath('image', '@src') + + item = l.load_item() + + assert item is l.item + assert item is nl1.item + assert item is nl2.item + + self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['url'], [u'http://www.scrapy.org']) + self.assertEqual(item['image'], [u'/images/logo.png']) + + class SelectJmesTestCase(unittest.TestCase): test_list_equals = { 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),