RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automated merge with http://hg.scrapy.org/scrapy-stable

This commit is contained in:
Pablo Hoffman 2009-10-02 09:21:21 -03:00
parent 0f1bb87198 64ad96ba1e
commit 9710f2635a
6 changed files with 108 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extras/build_release.sh
View File

@ -6,8 +6,8 @@
hg purge --all
# build packages
version=$(python -c "import scrapy; print scrapy.__version__")
python setup.py sdist
#version=$(python -c "import scrapy; print scrapy.__version__")
#python setup.py sdist
# FIXME: bdist_wininst doesn't work on Unix (it doesn't include the data_files)
# To build the win32 release you need to use Windows for now.
#python setup.py bdist_wininst -t "Scrapy $version" -p "win32"

4
scrapy/__init__.py
View File

@ -3,8 +3,8 @@ Scrapy - a screen scraping framework written in Python
"""
# IMPORTANT: remember to also update the version in docs/conf.py
version_info = (0, 7, 0, 'final', 0)
__version__ = "0.7"
version_info = (0, 8, 0, '', 0)
__version__ = "0.8-dev"
import sys, os

0
scrapy/contrib_exp/loader/__init__.py Normal file
View File

34
scrapy/contrib_exp/loader/lxmlloader.py Normal file
View File

@ -0,0 +1,34 @@
from lxml import html, etree
from scrapy.contrib.loader import ItemLoader
class LxmlItemLoader(ItemLoader):
def __init__(self, response, item=None, **context):
self.tree = html.fromstring(response.body_as_unicode())
context.update(response=response)
super(LxmlItemLoader, self).__init__(item, **context)
def add_xpath(self, field_name, xpath):
self.add_value(field_name, self._get_xpath(xpath))
def replace_xpath(self, field_name, xpath):
self.replace_value(field_name, self._get_xpath(xpath))
def _get_xpath(self, xpath):
return self._get_values(self.tree.xpath(xpath))
def add_css(self, field_name, css):
self.add_value(field_name, self._get_css(css))
def replace_css(self, field_name, css):
self.replace_value(field_name, self._get_css(css))
def _get_css(self, css):
return self._get_values(self.tree.cssselect(css))
def _get_values(self, elems):
for e in elems:
yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e

3
scrapy/core/engine.py
View File

@ -228,6 +228,9 @@ class ExecutionEngine(object):
self.next_request(spider)
return _
if spider not in self.downloader.sites:
return defer.fail(Failure(IgnoreRequest())).addBoth(_on_complete)
dwld = mustbe_deferred(self.downloader.fetch, request, spider)
dwld.addCallbacks(_on_success, _on_error)
dwld.addBoth(_on_complete)

67
scrapy/tests/test_contrib_exp_loader_lxmlloader.py Normal file
View File

@ -0,0 +1,67 @@
from twisted.trial import unittest
from scrapy.contrib.loader.processor import MapCompose
from scrapy.item import Item, Field
from scrapy.http import HtmlResponse
try:
import lxml
except ImportError:
lxml = False
class TestItem(Item):
name = Field()
if lxml:
from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader
class TestLxmlItemLoader(LxmlItemLoader):
default_item_class = TestItem
class LxmlItemLoaderTest(unittest.TestCase):
response = HtmlResponse(url="", body='<html><body><div id="id">marta</div><p>paragraph</p></body></html>')
def setUp(self):
if not lxml:
raise unittest.SkipTest("lxml is not available")
def test_constructor_with_response(self):
l = TestLxmlItemLoader(response=self.response)
self.assert_(l.tree)
def test_add_xpath(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
def test_add_xpath_text(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'marta'])
def test_replace_xpath(self):
l = TestLxmlItemLoader(response=self.response)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'marta'])
l.replace_xpath('name', '//p/text()')
self.assertEqual(l.get_output_value('name'), [u'paragraph'])
def test_add_css(self):
l = TestLxmlItemLoader(response=self.response)
l.add_css('name', '#id')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
def test_replace_css(self):
l = TestLxmlItemLoader(response=self.response)
l.add_css('name', '#id')
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
l.replace_css('name', 'p')
self.assertEqual(l.get_output_value('name'), [u'<p>paragraph</p>'])
if __name__ == "__main__":
unittest.main()