RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make libxml2 optional now that lxml is the default

This commit is contained in:
Daniel Graña 2012-05-16 18:17:51 -03:00
parent 8376d95ce8
commit f530b0b3eb
5 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/__init__.py
View File

@ -33,3 +33,10 @@ except ImportError:
pass
else:
optional_features.add('boto')
try:
import libxml2
except ImportError:
pass
else:
optional_features.add('libxml2')

16
scrapy/selector/libxml2document.py
View File

@ -4,16 +4,18 @@ garbage collection to libxml2 documents (xmlDoc).
"""
import weakref
import libxml2
from scrapy.utils.trackref import object_ref
from scrapy import optional_features
xml_parser_options = libxml2.XML_PARSE_RECOVER + \
libxml2.XML_PARSE_NOERROR + \
libxml2.XML_PARSE_NOWARNING
if 'libxml2' in optional_features:
import libxml2
xml_parser_options = libxml2.XML_PARSE_RECOVER + \
libxml2.XML_PARSE_NOERROR + \
libxml2.XML_PARSE_NOWARNING
html_parser_options = libxml2.HTML_PARSE_RECOVER + \
libxml2.HTML_PARSE_NOERROR + \
libxml2.HTML_PARSE_NOWARNING
html_parser_options = libxml2.HTML_PARSE_RECOVER + \
libxml2.HTML_PARSE_NOERROR + \
libxml2.HTML_PARSE_NOWARNING
_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8'))

4
scrapy/selector/libxml2sel.py
View File

@ -2,7 +2,9 @@
XPath selectors based on libxml2
"""
import libxml2
from scrapy import optional_features
if 'libxml2' in optional_features:
import libxml2
from scrapy.http import TextResponse
from scrapy.utils.python import unicode_to_str

7
scrapy/tests/test_libxml2.py
View File

@ -1,13 +1,12 @@
from twisted.trial import unittest
from scrapy.utils.test import libxml2debug
from scrapy import optional_features
class Libxml2Test(unittest.TestCase):
try:
import libxml2
except ImportError, e:
skip = str(e)
skip = 'libxml2' not in optional_features
@libxml2debug
def test_libxml2_bug_2_6_27(self):

8
scrapy/tests/test_selector_libxml2.py
View File

@ -2,7 +2,9 @@
Selectors tests, specific for libxml2 backend
"""
import unittest
from twisted.trial import unittest
from scrapy import optional_features
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \
@ -18,6 +20,8 @@ class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase):
hxs_cls = HtmlXPathSelector
xxs_cls = XmlXPathSelector
skip = 'libxml2' not in optional_features
@libxml2debug
def test_null_bytes(self):
hxs = HtmlXPathSelector(text='<root>la\x00la</root>')
@ -62,6 +66,8 @@ class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase):
class Libxml2DocumentTest(unittest.TestCase):
skip = 'libxml2' not in optional_features
@libxml2debug
def test_response_libxml2_caching(self):
r1 = HtmlResponse('http://www.example.com', body='<html><head></head><body></body></html>')