From 1f95af3c07da64621a86de643ffd993423baacd3 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Sat, 29 Aug 2015 12:04:00 -0700 Subject: [PATCH 1/5] add support for a nested loaders --- scrapy/loader/__init__.py | 41 +++++++++++++-- tests/test_loader.py | 108 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 4 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 84640f0b6..102ffc351 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -24,16 +24,46 @@ class ItemLoader(object): default_output_processor = Identity() default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, **context): + def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: selector = self.default_selector_class(response) self.selector = selector context.update(selector=selector, response=response) if item is None: item = self.default_item_class() - self.item = context['item'] = item self.context = context - self._values = defaultdict(list) + self.parent = parent + self._local_item = context['item'] = item + self._local_values = defaultdict(list) + + @property + def _values(self): + if self.parent is not None: + return self.parent._values + else: + return self._local_values + + @property + def item(self): + if self.parent is not None: + return self.parent.item + else: + return self._local_item + + def nested_loader(self, xpath=None, css=None): + if xpath is not None and css is not None: + raise ValueError("Cannot nest a loader with both a xpath selector and a css selector") + + if xpath is not None: + selector = self.selector.xpath(xpath) + + if css is not None: + selector = self.selector.css(css) + + subloader = self.__class__( + item=self.item, selector=selector, parent=self + ) + return subloader def add_value(self, field_name, value, *processors, **kw): value = self.get_value(value, *processors, **kw) @@ -84,6 +114,10 @@ class ItemLoader(object): value = self.get_output_value(field_name) if value is not None: item[field_name] = value + + # for loader in self._subloaders: + # loader.load_item() + return item def get_output_value(self, field_name): @@ -168,5 +202,4 @@ class ItemLoader(object): csss = arg_to_iter(csss) return flatten([self.selector.css(css).extract() for css in csss]) - XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8cf5e484a..2d34da810 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -19,11 +19,24 @@ class TestItem(NameItem): summary = Field() +class TestNestedItem(Item): + name = Field() + name_div = Field() + name_value = Field() + + url = Field() + image = Field() + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem +class NestedItemLoader(ItemLoader): + default_item_class = TestNestedItem + + class TestItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) +class SubselectorLoaderTest(unittest.TestCase): + response = HtmlResponse(url="", encoding='utf-8', body=b""" + + +
+
marta
+

paragraph

+
+ + + + """) + + def test_nested_xpath(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_loader(xpath="//header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_css(self): + l = NestedItemLoader(response=self.response) + nl = l.nested_loader(css="header") + nl.add_xpath('name', 'div/text()') + nl.add_css('name_div', '#id') + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + + self.assertEqual(l.get_output_value('name'), [u'marta']) + self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) + self.assertEqual(l.get_output_value('name_value'), [u'marta']) + + self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) + self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) + self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + + def test_nested_replace(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='a') + + l.add_xpath('url', '//footer/a/@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + nl1.replace_xpath('url', 'img/@src') + self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + nl2.replace_xpath('url', '@href') + self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + + def test_nested_ordering(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='a') + + nl1.add_xpath('url', 'img/@src') + l.add_xpath('url', '//footer/a/@href') + nl2.add_xpath('url', 'text()') + l.add_xpath('url', '//footer/a/@href') + + self.assertEqual(l.get_output_value('url'), [ + u'/images/logo.png', + u'http://www.scrapy.org', + u'homepage', + u'http://www.scrapy.org', + ]) + + def test_nested_load_item(self): + l = NestedItemLoader(response=self.response) + nl1 = l.nested_loader(xpath='//footer') + nl2 = nl1.nested_loader(xpath='img') + + l.add_xpath('name', '//header/div/text()') + nl1.add_xpath('url', 'a/@href') + nl2.add_xpath('image', '@src') + + item = l.load_item() + + assert item is l.item + assert item is nl1.item + assert item is nl2.item + + self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['url'], [u'http://www.scrapy.org']) + self.assertEqual(item['image'], [u'/images/logo.png']) + + class SelectJmesTestCase(unittest.TestCase): test_list_equals = { 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), From 425e35ee9023f5a973869d7e0d926297d9f75c35 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Sat, 29 Aug 2015 12:29:01 -0700 Subject: [PATCH 2/5] removed commented code and add test for error handling --- scrapy/loader/__init__.py | 3 --- tests/test_loader.py | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 102ffc351..f2c43ab4c 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -115,9 +115,6 @@ class ItemLoader(object): if value is not None: item[field_name] = value - # for loader in self._subloaders: - # loader.load_item() - return item def get_output_value(self, field_name): diff --git a/tests/test_loader.py b/tests/test_loader.py index 2d34da810..90273c2c8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -707,6 +707,11 @@ class SubselectorLoaderTest(unittest.TestCase): self.assertEqual(item['url'], [u'http://www.scrapy.org']) self.assertEqual(item['image'], [u'/images/logo.png']) + def test_nested_bad_arguments(self): + l = NestedItemLoader(response=self.response) + with self.assertRaises(ValueError): + l.nested_loader(css="#id", xpath="//footer") + class SelectJmesTestCase(unittest.TestCase): test_list_equals = { From 88c92cb68b5a6f9b0c50c801f92dacc4fc0fa52a Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Sat, 29 Aug 2015 14:23:25 -0700 Subject: [PATCH 3/5] provide documentation for nested loaders --- docs/topics/loaders.rst | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 8841711ea..0c20f32ef 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -432,6 +432,14 @@ ItemLoader objects ` to get the final value to assign to each item field. + .. method:: nested_loader(xpath=selector, css=selector) + + Create a nested loader with either an xpath selector or css selector. + The supplied selector is applied relative to selector associated + with this :class:`ItemLoader`. The nested loader shares the :class:`Item` + with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, + :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + .. method:: get_collected_values(field_name) Return the collected values for the given field. @@ -490,6 +498,52 @@ ItemLoader objects :attr:`default_selector_class`. This attribute is meant to be read-only. +.. _topics-loaders-nested: + +Nested Loaders +============== + +When parsing related values from a subsection of a document, it can be +useful to create nested loaders. Imagine you're extracting details from +a footer of a page that looks something like: + +Example:: + + + +Without nested loaders, you need to specify the full xpath (or css) for each value +that you wish to extract. + +Example:: + + loader = ItemLoader(item=Item()) + # load stuff not in the footer + loader.add_xpath('social', '//footer/a[@class = "social"]/@href') + loader.add_xpath('email', '//footer/a[@class = "email"]/@href') + loader.load_item() + +Instead, you can create a nested loader with the footer selector and add values +relative to the footer. The functionality is the same but you avoid repeating +the footer selector. + +Example:: + + loader = ItemLoader(item=Item()) + # load stuff not in the footer + footer_loader = loader.nested_loader(xpath='//footer') + footer_loader.add_xpath('social', 'a[@class = "social"]/@href') + footer_loader.add_xpath('email', 'a[@class = "email"]/@href') + # no need to call footer_loader.load_item() + loader.load_item() + +You can nest loaders arbitrarilly and they work with either xpath or css selectors. +As a general guideline, use nested loaders when they make your code simpler but do +not go overboard with nesting or your parser can become difficult to read. + .. _topics-loaders-extending: Reusing and extending Item Loaders From 311d5cd495c7bb82c39a85dd85438871518ea165 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Fri, 4 Sep 2015 12:19:10 -0700 Subject: [PATCH 4/5] split nested_loader into seperate methods --- scrapy/loader/__init__.py | 23 ++++++++++++----------- tests/test_loader.py | 21 ++++++++------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index f2c43ab4c..431f4d44b 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -50,18 +50,19 @@ class ItemLoader(object): else: return self._local_item - def nested_loader(self, xpath=None, css=None): - if xpath is not None and css is not None: - raise ValueError("Cannot nest a loader with both a xpath selector and a css selector") - - if xpath is not None: - selector = self.selector.xpath(xpath) - - if css is not None: - selector = self.selector.css(css) - + def nested_xpath(self, xpath, **context): + selector = self.selector.xpath(xpath) + context.update(selector=selector) subloader = self.__class__( - item=self.item, selector=selector, parent=self + item=self.item, parent=self, **context + ) + return subloader + + def nested_css(self, css, **context): + selector = self.selector.css(css) + context.update(selector=selector) + subloader = self.__class__( + item=self.item, parent=self, **context ) return subloader diff --git a/tests/test_loader.py b/tests/test_loader.py index 90273c2c8..2693a18d9 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -631,7 +631,7 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_xpath(self): l = NestedItemLoader(response=self.response) - nl = l.nested_loader(xpath="//header") + nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) @@ -646,7 +646,7 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_css(self): l = NestedItemLoader(response=self.response) - nl = l.nested_loader(css="header") + nl = l.nested_css("header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) @@ -661,8 +661,8 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_replace(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_loader(xpath='//footer') - nl2 = nl1.nested_loader(xpath='a') + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('a') l.add_xpath('url', '//footer/a/@href') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) @@ -673,8 +673,8 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_ordering(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_loader(xpath='//footer') - nl2 = nl1.nested_loader(xpath='a') + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('a') nl1.add_xpath('url', 'img/@src') l.add_xpath('url', '//footer/a/@href') @@ -690,8 +690,8 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_load_item(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_loader(xpath='//footer') - nl2 = nl1.nested_loader(xpath='img') + nl1 = l.nested_xpath('//footer') + nl2 = nl1.nested_xpath('img') l.add_xpath('name', '//header/div/text()') nl1.add_xpath('url', 'a/@href') @@ -707,11 +707,6 @@ class SubselectorLoaderTest(unittest.TestCase): self.assertEqual(item['url'], [u'http://www.scrapy.org']) self.assertEqual(item['image'], [u'/images/logo.png']) - def test_nested_bad_arguments(self): - l = NestedItemLoader(response=self.response) - with self.assertRaises(ValueError): - l.nested_loader(css="#id", xpath="//footer") - class SelectJmesTestCase(unittest.TestCase): test_list_equals = { From 036109e7ded7e186594c5c1ae642400e9e7d4b25 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 15 Sep 2015 23:49:35 -0700 Subject: [PATCH 5/5] updte nested loader documentation --- docs/topics/loaders.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 0c20f32ef..b188a0b81 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -432,9 +432,17 @@ ItemLoader objects ` to get the final value to assign to each item field. - .. method:: nested_loader(xpath=selector, css=selector) + .. method:: nested_xpath(xpath) - Create a nested loader with either an xpath selector or css selector. + Create a nested loader with an xpath selector. + The supplied selector is applied relative to selector associated + with this :class:`ItemLoader`. The nested loader shares the :class:`Item` + with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, + :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + + .. method:: nested_css(css) + + Create a nested loader with a css selector. The supplied selector is applied relative to selector associated with this :class:`ItemLoader`. The nested loader shares the :class:`Item` with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, @@ -534,7 +542,7 @@ Example:: loader = ItemLoader(item=Item()) # load stuff not in the footer - footer_loader = loader.nested_loader(xpath='//footer') + footer_loader = loader.nested_xpath('//footer') footer_loader.add_xpath('social', 'a[@class = "social"]/@href') footer_loader.add_xpath('email', 'a[@class = "email"]/@href') # no need to call footer_loader.load_item()