RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

{add,replace}_xpath: add processors, kw args and allow field_name to be None 

Also add method get_xpath.

Signed-off-by: Ping Yin <pkufranky@gmail.com>
This commit is contained in:
Ping Yin 2010-04-23 01:34:55 +08:00
parent a6c315552c
commit bd844f690b
2 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrapy/contrib/loader/__init__.py
View File

@ -124,13 +124,19 @@ class XPathItemLoader(ItemLoader):
context.update(selector=selector, response=response)
super(XPathItemLoader, self).__init__(item, **context)
def add_xpath(self, field_name, xpath, re=None):
self.add_value(field_name, self._get_values(field_name, xpath, re))
def add_xpath(self, field_name, xpath, *processors, **kw):
values = self._get_values(xpath, **kw)
self.add_value(field_name, values, *processors, **kw)
def replace_xpath(self, field_name, xpath, re=None):
self.replace_value(field_name, self._get_values(field_name, xpath, re))
def replace_xpath(self, field_name, xpath, *processors, **kw):
values = self._get_values(xpath, **kw)
self.replace_value(field_name, values, *processors, **kw)
def _get_values(self, field_name, xpath, re):
x = self.selector.select(xpath)
return x.re(re) if re else x.extract()
def get_xpath(self, xpath, *processors, **kw):
values = self._get_values(xpath, **kw)
return self.get_value(values, *processors, **kw)
def _get_values(self, xpaths, **kw):
xpaths = arg_to_iter(xpaths)
return flatten([self.selector.select(xpath).extract() for xpath in xpaths])

18
scrapy/tests/test_contrib_loader.py
View File

@ -360,6 +360,24 @@ class XPathItemLoaderTest(unittest.TestCase):
l.replace_xpath('name', '//p/text()')
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
l.replace_xpath('name', ['//p/text()', '//div/text()'])
self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
def test_get_xpath(self):
l = TestXPathItemLoader(response=self.response)
self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
def test_replace_xpath_multi_fields(self):
l = TestXPathItemLoader(response=self.response)
l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Marta'])
l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
def test_replace_xpath_re(self):
l = TestXPathItemLoader(response=self.response)
self.assert_(l.selector)