RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed LxmlItemLoader, as it has been obsoleted by the new lxml selector backend

This commit is contained in:
Pablo Hoffman 2010-10-30 16:02:14 -02:00
parent d67152ab0f
commit f73449fed6
3 changed files with 0 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
scrapy/contrib_exp/loader/__init__.py
View File

34
scrapy/contrib_exp/loader/lxmlloader.py
View File

@ -1,34 +0,0 @@
from lxml import html, etree
from scrapy.contrib.loader import ItemLoader
class LxmlItemLoader(ItemLoader):
def __init__(self, response, item=None, **context):
self.tree = html.fromstring(response.body_as_unicode())
context.update(response=response)
super(LxmlItemLoader, self).__init__(item, **context)
def add_xpath(self, field_name, xpath):
self.add_value(field_name, self._get_xpath(xpath))
def replace_xpath(self, field_name, xpath):
self.replace_value(field_name, self._get_xpath(xpath))
def _get_xpath(self, xpath):
return self._get_values(self.tree.xpath(xpath))
def add_css(self, field_name, css):
self.add_value(field_name, self._get_css(css))
def replace_css(self, field_name, css):
self.replace_value(field_name, self._get_css(css))
def _get_css(self, css):
return self._get_values(self.tree.cssselect(css))
def _get_values(self, elems):
for e in elems:
yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e

67
scrapy/tests/test_contrib_exp_loader_lxmlloader.py
View File

@ -1,67 +0,0 @@
from twisted.trial import unittest
from scrapy.contrib.loader.processor import MapCompose
from scrapy.item import Item, Field
from scrapy.http import HtmlResponse
try:
import lxml
except ImportError:
lxml = False
class TestItem(Item):
name = Field()
if lxml:
from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader
class TestLxmlItemLoader(LxmlItemLoader):
default_item_class = TestItem
class LxmlItemLoaderTest(unittest.TestCase):
response = HtmlResponse(url="", body='<html><body><div id="id">marta</div><p>paragraph</p></body></html>')
def setUp(self):
if not lxml:
raise unittest.SkipTest("lxml is not available")
def test_constructor_with_response(self):
l = TestLxmlItemLoader(response=self.response)
self.assert_(l.tree)
def test_add_xpath(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
def test_add_xpath_text(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'marta'])
def test_replace_xpath(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'marta'])
l.replace_xpath('name', '//p/text()')
self.assertEqual(l.get_output_value('name'), [u'paragraph'])
def test_add_css(self):
l = TestLxmlItemLoader(response=self.response)
l.add_css('name', '#id')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
def test_replace_css(self):
l = TestLxmlItemLoader(response=self.response)
l.add_css('name', '#id')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
l.replace_css('name', 'p')
self.assertEqual(l.get_output_value('name'), [u'<p>paragraph</p>'])
if __name__ == "__main__":
unittest.main()