From bd844f690be3221fb813f7de24b365cf5d13ba6f Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Fri, 23 Apr 2010 01:34:55 +0800 Subject: [PATCH] {add,replace}_xpath: add processors, kw args and allow field_name to be None Also add method get_xpath. Signed-off-by: Ping Yin --- scrapy/contrib/loader/__init__.py | 20 +++++++++++++------- scrapy/tests/test_contrib_loader.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 846f64165..445d72d57 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -124,13 +124,19 @@ class XPathItemLoader(ItemLoader): context.update(selector=selector, response=response) super(XPathItemLoader, self).__init__(item, **context) - def add_xpath(self, field_name, xpath, re=None): - self.add_value(field_name, self._get_values(field_name, xpath, re)) + def add_xpath(self, field_name, xpath, *processors, **kw): + values = self._get_values(xpath, **kw) + self.add_value(field_name, values, *processors, **kw) - def replace_xpath(self, field_name, xpath, re=None): - self.replace_value(field_name, self._get_values(field_name, xpath, re)) + def replace_xpath(self, field_name, xpath, *processors, **kw): + values = self._get_values(xpath, **kw) + self.replace_value(field_name, values, *processors, **kw) - def _get_values(self, field_name, xpath, re): - x = self.selector.select(xpath) - return x.re(re) if re else x.extract() + def get_xpath(self, xpath, *processors, **kw): + values = self._get_values(xpath, **kw) + return self.get_value(values, *processors, **kw) + + def _get_values(self, xpaths, **kw): + xpaths = arg_to_iter(xpaths) + return flatten([self.selector.select(xpath).extract() for xpath in xpaths]) diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 16f79ce75..612837ee8 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -360,6 +360,24 @@ class XPathItemLoaderTest(unittest.TestCase): l.replace_xpath('name', '//p/text()') self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + l.replace_xpath('name', ['//p/text()', '//div/text()']) + self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + + def test_get_xpath(self): + l = TestXPathItemLoader(response=self.response) + self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph']) + self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph') + self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa') + + self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta']) + + def test_replace_xpath_multi_fields(self): + l = TestXPathItemLoader(response=self.response) + l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) + self.assertEqual(l.get_output_value('name'), [u'Marta']) + l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) + self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + def test_replace_xpath_re(self): l = TestXPathItemLoader(response=self.response) self.assert_(l.selector)