RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implemented extract_unquote method over xpath selectors 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40221
This commit is contained in:
elpolilla 2008-09-11 11:14:44 +00:00
parent da7b87962a
commit d8b7f41ee5
2 changed files with 43 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
scrapy/trunk/scrapy/tests/test_xpath.py
View File

@ -191,5 +191,35 @@ class XPathTestCase(unittest.TestCase):
self.assertEqual(xxs.extract(),
u'<root>lala</root>')
if __name__ == "__main__":
unittest.main()
def test_unquote(self):
xmldoc='\n'.join((
'<root>',
' lala',
' <node>',
' blabla&amp;more<!--comment-->a<b>test</b>oh',
' <![CDATA[lalalal&ppppp<b>PPPP</b>ppp&amp;la]]>',
' </node>',
' pff',
'</root>'))
xxs = XmlXPathSelector(text=xmldoc)
self.assertEqual(xxs.extract_unquoted(), u'')
self.assertEqual(xxs.x('/root').extract_unquoted(), [u''])
self.assertEqual(xxs.x('/root/text()').extract_unquoted(), [
u'\n lala\n ',
u'\n pff\n'])
self.assertEqual(xxs.x('//*').extract_unquoted(), [u'', u'', u''])
self.assertEqual(xxs.x('//text()').extract_unquoted(), [
u'\n lala\n ',
u'\n blabla&more',
u'a',
u'test',
u'oh\n ',
u'lalalal&ppppp<b>PPPP</b>ppp&amp;la',
u'\n ',
u'\n pff\n'])
if __name__ == "__main__":
unittest.main()

12
scrapy/trunk/scrapy/xpath/selector.py
View File

@ -73,6 +73,13 @@ class XPathSelector(object):
text = unicode(self.xmlNode)
return text
def extract_unquoted(self):
"""Get unescaped contents from the text node (no entities, no CDATA)"""
if self.x('self::text()'):
return unicode(self.xmlNode.getContent(), errors='ignore')
else:
return u''
def register_namespace(self, prefix, uri):
"""Register namespace so that it can be used in XPath queries"""
self.doc.xpathContext.xpathRegisterNs(prefix, uri)
@ -98,12 +105,15 @@ class XPathSelectorList(list):
"""Perform the re() method on each XPathSelector of the list, and
return the result as a flattened list of unicode strings"""
return flatten([x.re(regex) for x in self])
def extract(self):
"""Return a list of unicode strings with the content referenced by each
XPathSelector of the list"""
return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
def extract_unquoted(self):
return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
class XmlXPathSelector(XPathSelector):
"""XPathSelector for XML content"""