From a59bfb539d5fb08a1a27a4bd63a753b571eaf40e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 25 Oct 2010 14:47:10 -0200 Subject: [PATCH] * Added lxml backend for XPath selectors. Closes #147 * Added new setting (SELECTORS_BACKEND) to choose which backend to use * Deprecated the extract_unquoted() function from selectors * Made libxml2 optional by adding a dummy selector backend. Closes #260 --HG-- rename : scrapy/tests/test_selector.py => scrapy/tests/test_selector_libxml2.py --- docs/topics/selectors.rst | 8 - scrapy/contrib/memdebug.py | 12 +- scrapy/selector/__init__.py | 171 ++---------- scrapy/selector/dummysel.py | 28 ++ scrapy/selector/libxml2sel.py | 151 ++++++++++ scrapy/selector/lxmlsel.py | 114 ++++++++ scrapy/settings/default_settings.py | 2 + scrapy/tests/test_libxml2.py | 20 ++ ...t_selector.py => test_selector_libxml2.py} | 13 - scrapy/tests/test_selector_lxml.py | 258 ++++++++++++++++++ scrapy/tests/test_utils_iterators.py | 1 - scrapy/utils/test.py | 5 +- 12 files changed, 609 insertions(+), 174 deletions(-) create mode 100644 scrapy/selector/dummysel.py create mode 100644 scrapy/selector/libxml2sel.py create mode 100644 scrapy/selector/lxmlsel.py create mode 100644 scrapy/tests/test_libxml2.py rename scrapy/tests/{test_selector.py => test_selector_libxml2.py} (96%) create mode 100644 scrapy/tests/test_selector_lxml.py diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cd3796296..9e9e47de9 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -255,14 +255,6 @@ XPathSelector objects Return a unicode string with the content of this :class:`XPathSelector` object. - .. method:: extract_unquoted() - - Return a unicode string with the content of this :class:`XPathSelector` - without entities or CDATA. This method is intended to be use for text-only - selectors, like ``//h1/text()`` (but not ``//h1``). If it's used for - :class:`XPathSelector` objects which don't select a textual content (ie. if - they contain tags), the output of this method is undefined. - .. method:: register_namespace(prefix, uri) Register the given namespace to be used in this :class:`XPathSelector`. diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index abc05cc4a..f2917e506 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -7,7 +7,6 @@ See documentation in docs/topics/extensions.rst import gc import socket -import libxml2 from scrapy.xlib.pydispatch import dispatcher from scrapy import signals @@ -19,6 +18,11 @@ from scrapy import log class MemoryDebugger(object): def __init__(self): + try: + import libxml2 + self.libxml2 = libxml2 + except ImportError: + raise NotConfigured if not settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured @@ -29,7 +33,7 @@ class MemoryDebugger(object): dispatcher.connect(self.engine_stopped, signals.engine_stopped) def engine_started(self): - libxml2.debugMemory(1) + self.libxml2.debugMemory(1) def engine_stopped(self): figures = self.collect_figures() @@ -37,12 +41,12 @@ class MemoryDebugger(object): self.log_or_send_report(report) def collect_figures(self): - libxml2.cleanupParser() + self.libxml2.cleanupParser() gc.collect() figures = [] figures.append(("Objects in gc.garbage", len(gc.garbage), "")) - figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes")) + figures.append(("libxml2 memory leak", self.libxml2.debugMemory(1), "bytes")) return figures def create_report(self, figures): diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index bf2dc52c7..d84c01f9f 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,153 +1,30 @@ """ -XPath selectors +XPath selectors -See documentation in docs/topics/selectors.rst +Two backends are currently available: libxml2 and lxml + +To select the backend explicitly use the SELECTORS_BACKEND variable in your +project. Otherwise, libxml2 will be tried first. If libxml2 is not available, +lxml will be used. """ -import libxml2 +from scrapy.conf import settings -from scrapy.http import TextResponse -from scrapy.utils.python import flatten, unicode_to_str -from scrapy.utils.misc import extract_regex -from scrapy.utils.trackref import object_ref -from scrapy.utils.decorator import deprecated -from .factories import xmlDoc_from_html, xmlDoc_from_xml -from .document import Libxml2Document - -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] - -class XPathSelector(object_ref): - - __slots__ = ['doc', 'xmlNode', 'expr', '__weakref__'] - - def __init__(self, response=None, text=None, node=None, parent=None, expr=None): - if parent: - self.doc = parent.doc - self.xmlNode = node - elif response: - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - elif text: - response = TextResponse(url='about:blank', \ - body=unicode_to_str(text, 'utf-8'), encoding='utf-8') - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - self.expr = expr - - def select(self, xpath): - """Perform the given XPath query on the current XPathSelector and - return a XPathSelectorList of the result""" - if hasattr(self.xmlNode, 'xpathEval'): - self.doc.xpathContext.setContextNode(self.xmlNode) - try: - xpath_result = self.doc.xpathContext.xpathEval(xpath) - except libxml2.xpathError: - raise ValueError("Invalid XPath: %s" % xpath) - if hasattr(xpath_result, '__iter__'): - return XPathSelectorList([self.__class__(node=node, parent=self, \ - expr=xpath) for node in xpath_result]) - else: - return XPathSelectorList([self.__class__(node=xpath_result, \ - parent=self, expr=xpath)]) +if settings['SELECTORS_BACKEND'] == 'lxml': + from .lxmlsel import * +elif settings['SELECTORS_BACKEND'] == 'libxml2': + from .libxml2sel import * +elif settings['SELECTORS_BACKEND'] == 'dummy': + from .dummysel import * +else: + try: + import libxml2 + except ImportError: + try: + import lxml + except ImportError: + from .dummysel import * else: - return XPathSelectorList([]) - - def re(self, regex): - """Return a list of unicode strings by applying the regex over all - current XPath selections, and flattening the results""" - return extract_regex(regex, self.extract(), 'utf-8') - - def extract(self): - """Return a unicode string of the content referenced by the XPathSelector""" - if isinstance(self.xmlNode, basestring): - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - elif hasattr(self.xmlNode, 'serialize'): - if isinstance(self.xmlNode, libxml2.xmlDoc): - data = self.xmlNode.getRootElement().serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - elif isinstance(self.xmlNode, libxml2.xmlAttr): - # serialization doesn't work sometimes for xmlAttr types - text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') - else: - data = self.xmlNode.serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - else: - try: - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - except TypeError: # catched when self.xmlNode is a float - see tests - text = unicode(self.xmlNode) - return text - - def extract_unquoted(self): - """Get unescaped contents from the text node (no entities, no CDATA)""" - if self.select('self::text()'): - return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') - else: - return u'' - - def register_namespace(self, prefix, uri): - """Register namespace so that it can be used in XPath queries""" - self.doc.xpathContext.xpathRegisterNs(prefix, uri) - - def _get_libxml2_doc(self, response): - """Return libxml2 document (xmlDoc) from response""" - return xmlDoc_from_html(response) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \ - 'name', type(self.xmlNode).__name__), self.expr) - - __repr__ = __str__ - - @deprecated(use_instead='XPathSelector.select') - def __call__(self, xpath): - return self.select(xpath) - - @deprecated(use_instead='XPathSelector.select') - def x(self, xpath): - return self.select(xpath) - - -class XPathSelectorList(list): - """List of XPathSelector objects""" - - def __getslice__(self, i, j): - return XPathSelectorList(list.__getslice__(self, i, j)) - - def select(self, xpath): - """Perform the given XPath query on each XPathSelector of the list and - return a new (flattened) XPathSelectorList of the results""" - return XPathSelectorList(flatten([x.select(xpath) for x in self])) - - def re(self, regex): - """Perform the re() method on each XPathSelector of the list, and - return the result as a flattened list of unicode strings""" - return flatten([x.re(regex) for x in self]) - - def extract(self): - """Return a list of unicode strings with the content referenced by each - XPathSelector of the list""" - return [x.extract() if isinstance(x, XPathSelector) else x for x in self] - - def extract_unquoted(self): - return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] - - @deprecated(use_instead='XPathSelectorList.select') - def x(self, xpath): - return self.select(xpath) - - -class XmlXPathSelector(XPathSelector): - """XPathSelector for XML content""" - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_xml) - - -class HtmlXPathSelector(XPathSelector): - """XPathSelector for HTML content""" - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_html) + from .lxmlsel import * + else: + from .libxml2sel import * diff --git a/scrapy/selector/dummysel.py b/scrapy/selector/dummysel.py new file mode 100644 index 000000000..7930835b6 --- /dev/null +++ b/scrapy/selector/dummysel.py @@ -0,0 +1,28 @@ +""" +Dummy selectors +""" + +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ + 'XPathSelectorList'] + +class XPathSelector(object): + + def __init__(self, *a, **kw): + pass + + def _raise(self, *a, **kw): + raise RuntimeError("No selectors backend available. " \ + "Please install libxml2 or lxml") + + select = re = exract = register_namespace = __nonzero__ = _raise + +class XPathSelectorList(list): + + def _raise(self, *a, **kw): + raise RuntimeError("No selectors backend available. " \ + "Please install libxml2 or lxml") + + __getslice__ = select = re = extract = extract_unquoted = _raise + +XmlXPathSelector = XPathSelector +HtmlXPathSelector = XPathSelector diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py new file mode 100644 index 000000000..f5fec4f93 --- /dev/null +++ b/scrapy/selector/libxml2sel.py @@ -0,0 +1,151 @@ +""" +XPath selectors based on libxml2 +""" + +import libxml2 + +from scrapy.http import TextResponse +from scrapy.utils.python import flatten, unicode_to_str +from scrapy.utils.misc import extract_regex +from scrapy.utils.trackref import object_ref +from scrapy.utils.decorator import deprecated +from .factories import xmlDoc_from_html, xmlDoc_from_xml +from .document import Libxml2Document + +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ + 'XPathSelectorList'] + +class XPathSelector(object_ref): + + __slots__ = ['doc', 'xmlNode', 'expr', '__weakref__'] + + def __init__(self, response=None, text=None, node=None, parent=None, expr=None): + if parent: + self.doc = parent.doc + self.xmlNode = node + elif response: + self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) + self.xmlNode = self.doc.xmlDoc + elif text: + response = TextResponse(url='about:blank', \ + body=unicode_to_str(text, 'utf-8'), encoding='utf-8') + self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) + self.xmlNode = self.doc.xmlDoc + self.expr = expr + + def select(self, xpath): + """Perform the given XPath query on the current XPathSelector and + return a XPathSelectorList of the result""" + if hasattr(self.xmlNode, 'xpathEval'): + self.doc.xpathContext.setContextNode(self.xmlNode) + try: + xpath_result = self.doc.xpathContext.xpathEval(xpath) + except libxml2.xpathError: + raise ValueError("Invalid XPath: %s" % xpath) + if hasattr(xpath_result, '__iter__'): + return XPathSelectorList([self.__class__(node=node, parent=self, \ + expr=xpath) for node in xpath_result]) + else: + return XPathSelectorList([self.__class__(node=xpath_result, \ + parent=self, expr=xpath)]) + else: + return XPathSelectorList([]) + + def re(self, regex): + """Return a list of unicode strings by applying the regex over all + current XPath selections, and flattening the results""" + return extract_regex(regex, self.extract(), 'utf-8') + + def extract(self): + """Return a unicode string of the content referenced by the XPathSelector""" + if isinstance(self.xmlNode, basestring): + text = unicode(self.xmlNode, 'utf-8', errors='ignore') + elif hasattr(self.xmlNode, 'serialize'): + if isinstance(self.xmlNode, libxml2.xmlDoc): + data = self.xmlNode.getRootElement().serialize('utf-8') + text = unicode(data, 'utf-8', errors='ignore') if data else u'' + elif isinstance(self.xmlNode, libxml2.xmlAttr): + # serialization doesn't work sometimes for xmlAttr types + text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') + else: + data = self.xmlNode.serialize('utf-8') + text = unicode(data, 'utf-8', errors='ignore') if data else u'' + else: + try: + text = unicode(self.xmlNode, 'utf-8', errors='ignore') + except TypeError: # catched when self.xmlNode is a float - see tests + text = unicode(self.xmlNode) + return text + + def extract_unquoted(self): + """Get unescaped contents from the text node (no entities, no CDATA)""" + if self.select('self::text()'): + return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') + else: + return u'' + + def register_namespace(self, prefix, uri): + """Register namespace so that it can be used in XPath queries""" + self.doc.xpathContext.xpathRegisterNs(prefix, uri) + + def _get_libxml2_doc(self, response): + """Return libxml2 document (xmlDoc) from response""" + return xmlDoc_from_html(response) + + def __nonzero__(self): + return bool(self.extract()) + + def __str__(self): + return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \ + 'name', type(self.xmlNode).__name__), self.expr) + + __repr__ = __str__ + + @deprecated(use_instead='XPathSelector.select') + def __call__(self, xpath): + return self.select(xpath) + + @deprecated(use_instead='XPathSelector.select') + def x(self, xpath): + return self.select(xpath) + + +class XPathSelectorList(list): + """List of XPathSelector objects""" + + def __getslice__(self, i, j): + return XPathSelectorList(list.__getslice__(self, i, j)) + + def select(self, xpath): + """Perform the given XPath query on each XPathSelector of the list and + return a new (flattened) XPathSelectorList of the results""" + return XPathSelectorList(flatten([x.select(xpath) for x in self])) + + def re(self, regex): + """Perform the re() method on each XPathSelector of the list, and + return the result as a flattened list of unicode strings""" + return flatten([x.re(regex) for x in self]) + + def extract(self): + """Return a list of unicode strings with the content referenced by each + XPathSelector of the list""" + return [x.extract() if isinstance(x, XPathSelector) else x for x in self] + + def extract_unquoted(self): + return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] + + @deprecated(use_instead='XPathSelectorList.select') + def x(self, xpath): + return self.select(xpath) + + +class XmlXPathSelector(XPathSelector): + """XPathSelector for XML content""" + __slots__ = () + _get_libxml2_doc = staticmethod(xmlDoc_from_xml) + + +class HtmlXPathSelector(XPathSelector): + """XPathSelector for HTML content""" + __slots__ = () + _get_libxml2_doc = staticmethod(xmlDoc_from_html) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py new file mode 100644 index 000000000..a313fad4e --- /dev/null +++ b/scrapy/selector/lxmlsel.py @@ -0,0 +1,114 @@ +""" +XPath selectors based on lxml +""" + +from lxml import etree + +from scrapy.utils.python import flatten +from scrapy.utils.misc import extract_regex +from scrapy.utils.trackref import object_ref +from scrapy.utils.python import unicode_to_str +from scrapy.utils.decorator import deprecated +from scrapy.http import TextResponse + +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ + 'XPathSelectorList'] + +class XPathSelector(object_ref): + + __slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '__weakref__'] + _parser = etree.HTMLParser + _tostring_method = 'html' + + def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None): + if text: + self.response = TextResponse(url='about:blank', \ + body=unicode_to_str(text, 'utf-8'), encoding='utf-8') + else: + self.response = response + self._root = root + self.namespaces = namespaces + self.expr = expr + + @property + def root(self): + if self._root is None: + parser = self._parser(encoding=self.response.encoding, recover=True) + self._root = etree.fromstring(self.response.body, parser=parser, \ + base_url=self.response.url) + return self._root + + def select(self, xpath): + xpatheval = etree.XPathEvaluator(self.root, namespaces=self.namespaces) + try: + result = xpatheval(xpath) + except etree.XPathError: + raise ValueError("Invalid XPath: %s" % xpath) + if hasattr(result, '__iter__'): + result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \ + for x in result] + elif result: + result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)] + return XPathSelectorList(result) + + def re(self, regex): + return extract_regex(regex, self.extract(), 'utf-8') + + def extract(self): + try: + return etree.tostring(self.root, method=self._tostring_method, \ + encoding=unicode).strip() + except (AttributeError, TypeError): + return unicode(self.root).strip() + + def register_namespace(self, prefix, uri): + if self.namespaces is None: + self.namespaces = {} + self.namespaces[prefix] = uri + + def __nonzero__(self): + return bool(self.extract()) + + def __str__(self): + data = repr(self.extract()[:40]) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) + + __repr__ = __str__ + + + @deprecated(use_instead='XPathSelector.extract') + def extract_unquoted(self): + return self.extract() + + +class XPathSelectorList(list): + + def __getslice__(self, i, j): + return XPathSelectorList(list.__getslice__(self, i, j)) + + def select(self, xpath): + return XPathSelectorList(flatten([x.select(xpath) for x in self])) + + def re(self, regex): + return flatten([x.re(regex) for x in self]) + + def extract(self): + return [x.extract() if isinstance(x, XPathSelector) else x for x in self] + + @deprecated(use_instead='XPathSelectorList.extract_unquoted') + def extract_unquoted(self): + return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] + + +class XmlXPathSelector(XPathSelector): + """XPathSelector for XML content""" + __slots__ = () + _parser = etree.XMLParser + _tostring_method = 'xml' + + +class HtmlXPathSelector(XPathSelector): + """XPathSelector for HTML content""" + __slots__ = () + _parser = etree.HTMLParser + _tostring_method = 'html' diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0b6703189..c48bc0fe6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -212,6 +212,8 @@ SCHEDULER_MIDDLEWARES_BASE = { SCHEDULER_ORDER = 'DFO' +SELECTORS_BACKEND = None # possible values: libxml2, lxml + SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager' SPIDER_MIDDLEWARES = {} diff --git a/scrapy/tests/test_libxml2.py b/scrapy/tests/test_libxml2.py new file mode 100644 index 000000000..3f3249440 --- /dev/null +++ b/scrapy/tests/test_libxml2.py @@ -0,0 +1,20 @@ +from twisted.trial import unittest + +from scrapy.utils.test import libxml2debug + +class Libxml2Test(unittest.TestCase): + + try: + import libxml2 + except ImportError, e: + skip = str(e) + + @libxml2debug + def test_libxml2_bug_2_6_27(self): + # this test will fail in version 2.6.27 but passes on 2.6.29+ + html = "123" + node = self.libxml2.htmlParseDoc(html, 'utf-8') + result = [str(r) for r in node.xpathEval('//text()')] + self.assertEquals(result, ['1', '2', '3']) + node.freeDoc() + diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector_libxml2.py similarity index 96% rename from scrapy/tests/test_selector.py rename to scrapy/tests/test_selector_libxml2.py index 8ceb87ed6..b40d0697d 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector_libxml2.py @@ -2,8 +2,6 @@ import re import unittest import weakref -import libxml2 - from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector @@ -284,16 +282,5 @@ class Libxml2DocumentTest(unittest.TestCase): headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) Libxml2Document(response) -class Libxml2Test(unittest.TestCase): - - @libxml2debug - def test_libxml2_bug_2_6_27(self): - # this test will fail in version 2.6.27 but passes on 2.6.29+ - html = "123" - node = libxml2.htmlParseDoc(html, 'utf-8') - result = [str(r) for r in node.xpathEval('//text()')] - self.assertEquals(result, ['1', '2', '3']) - node.freeDoc() - if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py new file mode 100644 index 000000000..2c977431f --- /dev/null +++ b/scrapy/tests/test_selector_lxml.py @@ -0,0 +1,258 @@ +# TODO: we should merge these tests with test_selector_libxml2.py + +import re +import unittest +import weakref + +from scrapy.http import TextResponse, HtmlResponse, XmlResponse +from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \ + XPathSelector +from scrapy.utils.test import libxml2debug + +class XPathSelectorTestCase(unittest.TestCase): + + @libxml2debug + def test_selector_simple(self): + """Simple selector tests""" + body = "

" + response = TextResponse(url="http://example.com", body=body) + xpath = HtmlXPathSelector(response) + + xl = xpath.select('//input') + self.assertEqual(2, len(xl)) + for x in xl: + assert isinstance(x, HtmlXPathSelector) + + self.assertEqual(xpath.select('//input').extract(), + [x.extract() for x in xpath.select('//input')]) + + self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], + [u'a']) + self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0']) + + self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), + [u'xpathrules']) + self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12']) + + @libxml2debug + def test_selector_same_type(self): + """Test XPathSelector returning the same type in x() method""" + text = '

test

' + assert isinstance(XmlXPathSelector(text=text).select("//p")[0], + XmlXPathSelector) + assert isinstance(HtmlXPathSelector(text=text).select("//p")[0], + HtmlXPathSelector) + + @libxml2debug + def test_selector_xml_html(self): + """Test that XML and HTML XPathSelector's behave differently""" + + # some text which is parsed differently by XML and HTML flavors + text = '

Hello

' + + self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(), + [u'

Hello

']) + + self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(), + [u'

Hello

']) + + @libxml2debug + def test_selector_nested(self): + """Nested selector tests""" + body = """ +
+ +
+
+ +
+ """ + + response = HtmlResponse(url="http://example.com", body=body) + x = HtmlXPathSelector(response) + + divtwo = x.select('//div[@class="two"]') + self.assertEqual(divtwo.select("//li").extract(), + ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select("./ul/li").extract(), + ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select(".//li").extract(), + ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select("./li").extract(), + []) + + @libxml2debug + def test_selector_re(self): + body = """
    Name: Mary + + Age: 20 +
    + + """ + response = HtmlResponse(url="http://example.com", body=body) + x = HtmlXPathSelector(response) + + name_re = re.compile("Name: (\w+)") + self.assertEqual(x.select("//ul/li").re(name_re), + ["John", "Paul"]) + self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), + ["10", "20"]) + + @libxml2debug + def test_selector_over_text(self): + hxs = HtmlXPathSelector(text='lala') + self.assertEqual(hxs.extract(), + u'lala') + + xxs = XmlXPathSelector(text='lala') + self.assertEqual(xxs.extract(), + u'lala') + + xxs = XmlXPathSelector(text='lala') + self.assertEqual(xxs.select('.').extract(), + [u'lala']) + + + @libxml2debug + def test_selector_namespaces_simple(self): + body = """ + + take this + found + + """ + + response = XmlResponse(url="http://example.com", body=body) + x = XmlXPathSelector(response) + + x.register_namespace("somens", "http://scrapy.org") + self.assertEqual(x.select("//somens:a/text()").extract(), + [u'take this']) + + + @libxml2debug + def test_selector_namespaces_multiple(self): + body = """ + + hello + value + iron90Dried Rose + + """ + response = XmlResponse(url="http://example.com", body=body) + x = XmlXPathSelector(response) + + x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") + x.register_namespace("p", "http://www.scrapy.org/product") + x.register_namespace("b", "http://somens.com") + self.assertEqual(len(x.select("//xmlns:TestTag")), 1) + self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') + + @libxml2debug + def test_selector_invalid_xpath(self): + response = XmlResponse(url="http://example.com", body="") + x = HtmlXPathSelector(response) + xpath = "//test[@foo='bar]" + try: + x.select(xpath) + except ValueError, e: + assert xpath in str(e), "Exception message does not contain invalid xpath" + except Exception: + raise AssertionError("A invalid XPath does not raise ValueError") + else: + raise AssertionError("A invalid XPath does not raise an exception") + + @libxml2debug + def test_http_header_encoding_precedence(self): + # u'\xa3' = pound symbol in unicode + # u'\xc2\xa3' = pound symbol in utf-8 + # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + + meta = u'' + head = u'' + meta + u'' + body_content = u'\xa3' + body = u'' + body_content + u'' + html = u'' + head + body + u'' + encoding = 'utf-8' + html_utf8 = html.encode(encoding) + + headers = {'Content-Type': ['text/html; charset=utf-8']} + response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) + x = HtmlXPathSelector(response) + self.assertEquals(x.select("//span[@id='blank']/text()").extract(), + [u'\xa3']) + + @libxml2debug + def test_null_bytes(self): + hxs = HtmlXPathSelector(text='la\x00la') + self.assertEqual(hxs.extract(), + u'la la') + + xxs = XmlXPathSelector(text='la\x00la') + self.assertEqual(xxs.extract(), + u'la') + + @libxml2debug + def test_unquote(self): + xmldoc = '\n'.join(( + '', + ' lala', + ' ', + ' blabla&moreatestoh', + ' PPPPppp&la]]>', + ' ', + ' pff', + '')) + xxs = XmlXPathSelector(text=xmldoc) + + # this tests were commented out because they make no sense (pablo) + #self.assertEqual(xxs.extract_unquoted(), u'') + #self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) + #self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) + + self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ + u'lala', + u'pff']) + + self.assertEqual(xxs.select('//text()').extract_unquoted(), [ + u'lala', + u'blabla&more', + u'a', + u'test', + u'oh\n lalalal&pppppPPPPppp&la', + u'pff']) + + @libxml2debug + def test_empty_bodies(self): + r1 = TextResponse('http://www.example.com', body='') + hxs = HtmlXPathSelector(r1) # shouldn't raise error + xxs = XmlXPathSelector(r1) # shouldn't raise error + + @libxml2debug + def test_weakref_slots(self): + """Check that classes are using slots and are weak-referenceable""" + for cls in [XPathSelector, HtmlXPathSelector, XmlXPathSelector]: + x = cls() + weakref.ref(x) + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ + x.__class__.__name__ + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index ffbe27262..0c77ded21 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -1,5 +1,4 @@ import os -import libxml2 from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 597ee909c..87cbc3ab8 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,7 +4,6 @@ This module contains some assorted functions used in tests import os -import libxml2 from twisted.trial.unittest import SkipTest from scrapy.crawler import Crawler @@ -19,6 +18,10 @@ def libxml2debug(testfunction): LIBXML2_DEBUGLEAKS is set. """ + try: + import libxml2 + except ImportError: + return testfunction def newfunc(*args, **kwargs): libxml2.debugMemory(1) testfunction(*args, **kwargs)