diff --git a/extras/build_release.sh b/extras/build_release.sh
index 57873b12e..2230685e8 100755
--- a/extras/build_release.sh
+++ b/extras/build_release.sh
@@ -6,8 +6,8 @@
hg purge --all
# build packages
-version=$(python -c "import scrapy; print scrapy.__version__")
-python setup.py sdist
+#version=$(python -c "import scrapy; print scrapy.__version__")
+#python setup.py sdist
# FIXME: bdist_wininst doesn't work on Unix (it doesn't include the data_files)
# To build the win32 release you need to use Windows for now.
#python setup.py bdist_wininst -t "Scrapy $version" -p "win32"
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index bee059b0b..5e206de2f 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -3,8 +3,8 @@ Scrapy - a screen scraping framework written in Python
"""
# IMPORTANT: remember to also update the version in docs/conf.py
-version_info = (0, 7, 0, 'final', 0)
-__version__ = "0.7"
+version_info = (0, 8, 0, '', 0)
+__version__ = "0.8-dev"
import sys, os
diff --git a/scrapy/contrib_exp/loader/__init__.py b/scrapy/contrib_exp/loader/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/contrib_exp/loader/lxmlloader.py b/scrapy/contrib_exp/loader/lxmlloader.py
new file mode 100644
index 000000000..724e7c43c
--- /dev/null
+++ b/scrapy/contrib_exp/loader/lxmlloader.py
@@ -0,0 +1,34 @@
+from lxml import html, etree
+
+from scrapy.contrib.loader import ItemLoader
+
+
+class LxmlItemLoader(ItemLoader):
+
+ def __init__(self, response, item=None, **context):
+ self.tree = html.fromstring(response.body_as_unicode())
+ context.update(response=response)
+ super(LxmlItemLoader, self).__init__(item, **context)
+
+ def add_xpath(self, field_name, xpath):
+ self.add_value(field_name, self._get_xpath(xpath))
+
+ def replace_xpath(self, field_name, xpath):
+ self.replace_value(field_name, self._get_xpath(xpath))
+
+ def _get_xpath(self, xpath):
+ return self._get_values(self.tree.xpath(xpath))
+
+ def add_css(self, field_name, css):
+ self.add_value(field_name, self._get_css(css))
+
+ def replace_css(self, field_name, css):
+ self.replace_value(field_name, self._get_css(css))
+
+ def _get_css(self, css):
+ return self._get_values(self.tree.cssselect(css))
+
+ def _get_values(self, elems):
+ for e in elems:
+ yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e
+
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 004a65b98..6f1352d6a 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -228,6 +228,9 @@ class ExecutionEngine(object):
self.next_request(spider)
return _
+ if spider not in self.downloader.sites:
+ return defer.fail(Failure(IgnoreRequest())).addBoth(_on_complete)
+
dwld = mustbe_deferred(self.downloader.fetch, request, spider)
dwld.addCallbacks(_on_success, _on_error)
dwld.addBoth(_on_complete)
diff --git a/scrapy/tests/test_contrib_exp_loader_lxmlloader.py b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py
new file mode 100644
index 000000000..1980648c1
--- /dev/null
+++ b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py
@@ -0,0 +1,67 @@
+from twisted.trial import unittest
+
+from scrapy.contrib.loader.processor import MapCompose
+from scrapy.item import Item, Field
+from scrapy.http import HtmlResponse
+
+try:
+ import lxml
+except ImportError:
+ lxml = False
+
+
+class TestItem(Item):
+ name = Field()
+
+
+if lxml:
+ from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader
+
+ class TestLxmlItemLoader(LxmlItemLoader):
+ default_item_class = TestItem
+
+
+class LxmlItemLoaderTest(unittest.TestCase):
+ response = HtmlResponse(url="", body='marta
paragraph
')
+
+ def setUp(self):
+ if not lxml:
+ raise unittest.SkipTest("lxml is not available")
+
+ def test_constructor_with_response(self):
+ l = TestLxmlItemLoader(response=self.response)
+ self.assert_(l.tree)
+
+ def test_add_xpath(self):
+ l = TestLxmlItemLoader(response=self.response)
+ l.add_xpath('name', '//div')
+ self.assertEqual(l.get_output_value('name'), [u'marta
'])
+
+ def test_add_xpath_text(self):
+ l = TestLxmlItemLoader(response=self.response)
+ l.add_xpath('name', '//div/text()')
+ self.assertEqual(l.get_output_value('name'), [u'marta'])
+
+ def test_replace_xpath(self):
+ l = TestLxmlItemLoader(response=self.response)
+ l.add_xpath('name', '//div/text()')
+ self.assertEqual(l.get_output_value('name'), [u'marta'])
+ l.replace_xpath('name', '//p/text()')
+ self.assertEqual(l.get_output_value('name'), [u'paragraph'])
+
+ def test_add_css(self):
+ l = TestLxmlItemLoader(response=self.response)
+ l.add_css('name', '#id')
+ self.assertEqual(l.get_output_value('name'), [u'marta
'])
+
+ def test_replace_css(self):
+ l = TestLxmlItemLoader(response=self.response)
+ l.add_css('name', '#id')
+ self.assertEqual(l.get_output_value('name'), [u'marta
'])
+ l.replace_css('name', 'p')
+ self.assertEqual(l.get_output_value('name'), [u'paragraph
'])
+
+
+if __name__ == "__main__":
+ unittest.main()
+