Merge pull request #1467 from dacjames/master

add support for a nested loaders

docs/topics/loaders.rst

@@ -432,6 +432,22 @@ ItemLoader objects
         <topics-loaders-processors>` to get the final value to assign to each
         item field.
 
+    .. method:: nested_xpath(xpath)
+
+        Create a nested loader with an xpath selector.
+        The supplied selector is applied relative to selector associated
+        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
+        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
+        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+
+    .. method:: nested_css(css)
+
+        Create a nested loader with a css selector.
+        The supplied selector is applied relative to selector associated
+        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
+        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
+        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+
     .. method:: get_collected_values(field_name)
 
         Return the collected values for the given field.
@@ -490,6 +506,52 @@ ItemLoader objects
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
+.. _topics-loaders-nested:
+
+Nested Loaders
+==============
+
+When parsing related values from a subsection of a document, it can be
+useful to create nested loaders.  Imagine you're extracting details from
+a footer of a page that looks something like:
+
+Example::
+
+    <footer>
+        <a class="social" href="http://facebook.com/whatever">Like Us</a>
+        <a class="social" href="http://twitter.com/whatever">Follow Us</a>
+        <a class="email" href="mailto:whatever@example.com">Email Us</a>
+    </footer>
+
+Without nested loaders, you need to specify the full xpath (or css) for each value
+that you wish to extract.
+
+Example::
+
+    loader = ItemLoader(item=Item())
+    # load stuff not in the footer
+    loader.add_xpath('social', '//footer/a[@class = "social"]/@href')
+    loader.add_xpath('email', '//footer/a[@class = "email"]/@href')
+    loader.load_item()
+
+Instead, you can create a nested loader with the footer selector and add values
+relative to the footer.  The functionality is the same but you avoid repeating
+the footer selector.
+
+Example::
+
+    loader = ItemLoader(item=Item())
+    # load stuff not in the footer
+    footer_loader = loader.nested_xpath('//footer')
+    footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
+    footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
+    # no need to call footer_loader.load_item()
+    loader.load_item()
+
+You can nest loaders arbitrarilly and they work with either xpath or css selectors.
+As a general guideline, use nested loaders when they make your code simpler but do
+not go overboard with nesting or your parser can become difficult to read.
+
 .. _topics-loaders-extending:
 
 Reusing and extending Item Loaders

scrapy/loader/__init__.py

@@ -24,16 +24,47 @@ class ItemLoader(object):
     default_output_processor = Identity()
     default_selector_class = Selector
 
-    def __init__(self, item=None, selector=None, response=None, **context):
+    def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
             selector = self.default_selector_class(response)
         self.selector = selector
         context.update(selector=selector, response=response)
         if item is None:
             item = self.default_item_class()
-        self.item = context['item'] = item
         self.context = context
-        self._values = defaultdict(list)
+        self.parent = parent
+        self._local_item = context['item'] = item
+        self._local_values = defaultdict(list)
+
+    @property
+    def _values(self):
+        if self.parent is not None:
+            return self.parent._values
+        else:
+            return self._local_values
+
+    @property
+    def item(self):
+        if self.parent is not None:
+            return self.parent.item
+        else:
+            return self._local_item
+
+    def nested_xpath(self, xpath, **context):
+        selector = self.selector.xpath(xpath)
+        context.update(selector=selector)
+        subloader = self.__class__(
+            item=self.item, parent=self, **context
+        )
+        return subloader
+
+    def nested_css(self, css, **context):
+        selector = self.selector.css(css)
+        context.update(selector=selector)
+        subloader = self.__class__(
+            item=self.item, parent=self, **context
+        )
+        return subloader
 
     def add_value(self, field_name, value, *processors, **kw):
         value = self.get_value(value, *processors, **kw)
@@ -84,6 +115,7 @@ class ItemLoader(object):
             value = self.get_output_value(field_name)
             if value is not None:
                 item[field_name] = value
+
         return item
 
     def get_output_value(self, field_name):
@@ -168,5 +200,4 @@ class ItemLoader(object):
         csss = arg_to_iter(csss)
         return flatten(self.selector.css(css).extract() for css in csss)
 
-
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

tests/test_loader.py

@@ -19,11 +19,24 @@ class TestItem(NameItem):
     summary = Field()
 
 
+class TestNestedItem(Item):
+    name = Field()
+    name_div = Field()
+    name_value = Field()
+
+    url = Field()
+    image = Field()
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
 
 
+class NestedItemLoader(ItemLoader):
+    default_item_class = TestNestedItem
+
+
 class TestItemLoader(NameItemLoader):
     name_in = MapCompose(lambda v: v.title())
 
@@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 
+class SubselectorLoaderTest(unittest.TestCase):
+    response = HtmlResponse(url="", encoding='utf-8', body=b"""
+    <html>
+    <body>
+    <header>
+      <div id="id">marta</div>
+      <p>paragraph</p>
+    </header>
+    <footer class="footer">
+      <a href="http://www.scrapy.org">homepage</a>
+      <img src="/images/logo.png" width="244" height="65" alt="Scrapy">
+    </footer>
+    </body>
+    </html>
+    """)
+
+    def test_nested_xpath(self):
+        l = NestedItemLoader(response=self.response)
+        nl = l.nested_xpath("//header")
+        nl.add_xpath('name', 'div/text()')
+        nl.add_css('name_div', '#id')
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+
+        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
+        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
+        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+
+    def test_nested_css(self):
+        l = NestedItemLoader(response=self.response)
+        nl = l.nested_css("header")
+        nl.add_xpath('name', 'div/text()')
+        nl.add_css('name_div', '#id')
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+
+        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
+        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
+        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+
+    def test_nested_replace(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('a')
+
+        l.add_xpath('url', '//footer/a/@href')
+        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        nl1.replace_xpath('url', 'img/@src')
+        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        nl2.replace_xpath('url', '@href')
+        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+
+    def test_nested_ordering(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('a')
+
+        nl1.add_xpath('url', 'img/@src')
+        l.add_xpath('url', '//footer/a/@href')
+        nl2.add_xpath('url', 'text()')
+        l.add_xpath('url', '//footer/a/@href')
+
+        self.assertEqual(l.get_output_value('url'), [
+            u'/images/logo.png',
+            u'http://www.scrapy.org',
+            u'homepage',
+            u'http://www.scrapy.org',
+        ])
+
+    def test_nested_load_item(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('img')
+
+        l.add_xpath('name', '//header/div/text()')
+        nl1.add_xpath('url', 'a/@href')
+        nl2.add_xpath('image', '@src')
+
+        item = l.load_item()
+
+        assert item is l.item
+        assert item is nl1.item
+        assert item is nl2.item
+
+        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['url'], [u'http://www.scrapy.org'])
+        self.assertEqual(item['image'], [u'/images/logo.png'])
+
+
 class SelectJmesTestCase(unittest.TestCase):
         test_list_equals = {
             'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),