From 12ebcfd6c4100be970a9e140c52221305f38eae3 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 18 Sep 2009 15:31:54 -0300 Subject: [PATCH 1/4] bumped version to 0.8 --- scrapy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f9f348332..c4472cf8a 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -3,8 +3,8 @@ Scrapy - a screen scraping framework written in Python """ # IMPORTANT: remember to also update the version in docs/conf.py -version_info = (0, 7, 0, 'candidate', 0) -__version__ = "0.7.0-rc1" +version_info = (0, 8, 0, '', 0) +__version__ = "0.8.0-dev" import sys, os From 201ce525e51372e5beb6b0c724d1ab76f33c6cb1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 21 Sep 2009 13:07:52 -0300 Subject: [PATCH 2/4] make_release.sh: disabled bdist_wininst command as it doesn't package data files properly on Linux - refs #109 --- extras/make_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extras/make_release.sh b/extras/make_release.sh index 08730006c..0f8f3dbc3 100755 --- a/extras/make_release.sh +++ b/extras/make_release.sh @@ -6,8 +6,8 @@ hg purge --all # build packages -version=$(python -c "import scrapy; print scrapy.__version__") -python setup.py sdist +#version=$(python -c "import scrapy; print scrapy.__version__") +#python setup.py sdist # FIXME: bdist_wininst doesn't work on Unix (it doesn't include the data_files) #python setup.py bdist_wininst -t "Scrapy $version" -p "win32" From e0ccb2cceb97adda9ffd7da971e2fb18fb5a300f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 28 Sep 2009 12:44:05 -0300 Subject: [PATCH 3/4] ignore request if spider is not opened in downloader when called from engine.download --- scrapy/core/engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 004a65b98..6f1352d6a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -228,6 +228,9 @@ class ExecutionEngine(object): self.next_request(spider) return _ + if spider not in self.downloader.sites: + return defer.fail(Failure(IgnoreRequest())).addBoth(_on_complete) + dwld = mustbe_deferred(self.downloader.fetch, request, spider) dwld.addCallbacks(_on_success, _on_error) dwld.addBoth(_on_complete) From 64ad96ba1ead3c670820db828d6155fd3d082ffa Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Tue, 29 Sep 2009 16:17:30 -0300 Subject: [PATCH 4/4] added LxmlItemLoader --- scrapy/contrib_exp/loader/__init__.py | 0 scrapy/contrib_exp/loader/lxmlloader.py | 34 ++++++++++ .../test_contrib_exp_loader_lxmlloader.py | 67 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 scrapy/contrib_exp/loader/__init__.py create mode 100644 scrapy/contrib_exp/loader/lxmlloader.py create mode 100644 scrapy/tests/test_contrib_exp_loader_lxmlloader.py diff --git a/scrapy/contrib_exp/loader/__init__.py b/scrapy/contrib_exp/loader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/contrib_exp/loader/lxmlloader.py b/scrapy/contrib_exp/loader/lxmlloader.py new file mode 100644 index 000000000..724e7c43c --- /dev/null +++ b/scrapy/contrib_exp/loader/lxmlloader.py @@ -0,0 +1,34 @@ +from lxml import html, etree + +from scrapy.contrib.loader import ItemLoader + + +class LxmlItemLoader(ItemLoader): + + def __init__(self, response, item=None, **context): + self.tree = html.fromstring(response.body_as_unicode()) + context.update(response=response) + super(LxmlItemLoader, self).__init__(item, **context) + + def add_xpath(self, field_name, xpath): + self.add_value(field_name, self._get_xpath(xpath)) + + def replace_xpath(self, field_name, xpath): + self.replace_value(field_name, self._get_xpath(xpath)) + + def _get_xpath(self, xpath): + return self._get_values(self.tree.xpath(xpath)) + + def add_css(self, field_name, css): + self.add_value(field_name, self._get_css(css)) + + def replace_css(self, field_name, css): + self.replace_value(field_name, self._get_css(css)) + + def _get_css(self, css): + return self._get_values(self.tree.cssselect(css)) + + def _get_values(self, elems): + for e in elems: + yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e + diff --git a/scrapy/tests/test_contrib_exp_loader_lxmlloader.py b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py new file mode 100644 index 000000000..1980648c1 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py @@ -0,0 +1,67 @@ +from twisted.trial import unittest + +from scrapy.contrib.loader.processor import MapCompose +from scrapy.item import Item, Field +from scrapy.http import HtmlResponse + +try: + import lxml +except ImportError: + lxml = False + + +class TestItem(Item): + name = Field() + + +if lxml: + from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader + + class TestLxmlItemLoader(LxmlItemLoader): + default_item_class = TestItem + + +class LxmlItemLoaderTest(unittest.TestCase): + response = HtmlResponse(url="", body='
marta

paragraph

') + + def setUp(self): + if not lxml: + raise unittest.SkipTest("lxml is not available") + + def test_constructor_with_response(self): + l = TestLxmlItemLoader(response=self.response) + self.assert_(l.tree) + + def test_add_xpath(self): + l = TestLxmlItemLoader(response=self.response) + l.add_xpath('name', '//div') + self.assertEqual(l.get_output_value('name'), [u'
marta
']) + + def test_add_xpath_text(self): + l = TestLxmlItemLoader(response=self.response) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'marta']) + + def test_replace_xpath(self): + l = TestLxmlItemLoader(response=self.response) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'marta']) + l.replace_xpath('name', '//p/text()') + self.assertEqual(l.get_output_value('name'), [u'paragraph']) + + def test_add_css(self): + l = TestLxmlItemLoader(response=self.response) + l.add_css('name', '#id') + self.assertEqual(l.get_output_value('name'), [u'
marta
']) + + def test_replace_css(self): + l = TestLxmlItemLoader(response=self.response) + l.add_css('name', '#id') + self.assertEqual(l.get_output_value('name'), [u'
marta
']) + l.replace_css('name', 'p') + self.assertEqual(l.get_output_value('name'), [u'

paragraph

']) + + +if __name__ == "__main__": + unittest.main() +