diff --git a/docs/news.rst b/docs/news.rst index 8d6022fe7..dd4f571b6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,6 +9,8 @@ Release notes - Request/Response url/body attributes are now immutable (modifying them had been deprecated for a long time) - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) +- Dropped libxml2 selectors backend +- Dropped support for multiple selectors backends, sticking to lxml only 0.18.4 (released 2013-10-10) ---------------------------- diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 9a4c0bf2c..eb944fa34 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -248,7 +248,6 @@ Memory debugger extension An extension for debugging memory usage. It collects information about: * objects uncollected by the Python garbage collector -* libxml2 memory leaks * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cd4d825d9..554f16eb6 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -40,7 +40,6 @@ reference ` and :ref:`CSS selector reference .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://codespeak.net/lxml/ .. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html -.. _libxml2: http://xmlsoft.org/ .. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors diff --git a/scrapy/__init__.py b/scrapy/__init__.py index e32618ebb..fef482d52 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -30,13 +30,6 @@ except ImportError: else: optional_features.add('boto') -try: - import libxml2 -except ImportError: - pass -else: - optional_features.add('libxml2') - try: import django except ImportError: diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 27e12046a..36210a04b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -6,6 +6,7 @@ import twisted import scrapy from scrapy.command import ScrapyCommand + class Command(ScrapyCommand): def syntax(self): @@ -21,13 +22,9 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - try: - import lxml.etree - except ImportError: - lxml_version = libxml2_version = "(lxml not available)" - else: - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + import lxml.etree + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) print "Scrapy : %s" % scrapy.__version__ print "lxml : %s" % lxml_version print "libxml2 : %s" % libxml2_version diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index 359c7ec8a..2d9249f96 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -10,14 +10,10 @@ from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs + class MemoryDebugger(object): def __init__(self, stats): - try: - import libxml2 - self.libxml2 = libxml2 - except ImportError: - self.libxml2 = None self.stats = stats @classmethod @@ -25,18 +21,10 @@ class MemoryDebugger(object): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured o = cls(crawler.stats) - crawler.signals.connect(o.engine_started, signals.engine_started) crawler.signals.connect(o.engine_stopped, signals.engine_stopped) return o - def engine_started(self): - if self.libxml2: - self.libxml2.debugMemory(1) - def engine_stopped(self): - if self.libxml2: - self.libxml2.cleanupParser() - self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) for cls, wdict in live_refs.iteritems(): diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 2b855d147..7dd420d1d 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,29 +1,7 @@ """ -XPath selectors - -To select the backend explicitly use the SCRAPY_SELECTORS_BACKEND environment -variable. - -Two backends are currently available: lxml (default) and libxml2. - +Selectors """ -import os - - -backend = os.environ.get('SCRAPY_SELECTORS_BACKEND') -if backend == 'libxml2': - from scrapy.selector.libxml2sel import * -elif backend == 'lxml': - from scrapy.selector.lxmlsel import * -else: - try: - import lxml - except ImportError: - import libxml2 - from scrapy.selector.libxml2sel import * - else: - from scrapy.selector.lxmlsel import * - +from scrapy.selector.lxmlsel import * from scrapy.selector.csssel import * from scrapy.selector.list import SelectorList diff --git a/scrapy/selector/libxml2document.py b/scrapy/selector/libxml2document.py deleted file mode 100644 index 070dd0c58..000000000 --- a/scrapy/selector/libxml2document.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -This module contains a simple class (Libxml2Document) which provides cache and -garbage collection to libxml2 documents (xmlDoc). -""" - -import weakref -from scrapy.utils.trackref import object_ref -from scrapy import optional_features - -if 'libxml2' in optional_features: - import libxml2 - xml_parser_options = libxml2.XML_PARSE_RECOVER + \ - libxml2.XML_PARSE_NOERROR + \ - libxml2.XML_PARSE_NOWARNING - - html_parser_options = libxml2.HTML_PARSE_RECOVER + \ - libxml2.HTML_PARSE_NOERROR + \ - libxml2.HTML_PARSE_NOWARNING - - -_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) -def _body_as_utf8(response): - if response.encoding in _UTF8_ENCODINGS: - return response.body - else: - return response.body_as_unicode().encode('utf-8') - - -def xmlDoc_from_html(response): - """Return libxml2 doc for HTMLs""" - utf8body = _body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ - html_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', html_parser_options) - return lxdoc - - -def xmlDoc_from_xml(response): - """Return libxml2 doc for XMLs""" - utf8body = _body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ - xml_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', xml_parser_options) - return lxdoc - - -class Libxml2Document(object_ref): - - cache = weakref.WeakKeyDictionary() - __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] - - def __new__(cls, response, factory=xmlDoc_from_html): - cache = cls.cache.setdefault(response, {}) - if factory not in cache: - obj = object_ref.__new__(cls) - obj.xmlDoc = factory(response) - obj.xpathContext = obj.xmlDoc.xpathNewContext() - cache[factory] = obj - return cache[factory] - - def __del__(self): - # we must call both cleanup functions, so we try/except all exceptions - # to make sure one doesn't prevent the other from being called - # this call sometimes raises a "NoneType is not callable" TypeError - # so the try/except block silences them - try: - self.xmlDoc.freeDoc() - except: - pass - try: - self.xpathContext.xpathFreeContext() - except: - pass - - def __str__(self): - return "" % self.xmlDoc.name diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py deleted file mode 100644 index 2f5d72ea1..000000000 --- a/scrapy/selector/libxml2sel.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -XPath selectors based on libxml2 -""" - -from scrapy import optional_features -if 'libxml2' in optional_features: - import libxml2 - -from scrapy.http import TextResponse -from scrapy.utils.python import unicode_to_str -from scrapy.utils.misc import extract_regex -from scrapy.utils.trackref import object_ref -from scrapy.utils.decorator import deprecated -from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml -from .list import SelectorList - - -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] - - -class XPathSelector(object_ref): - - __slots__ = ['doc', 'xmlNode', 'expr', '__weakref__'] - - def __init__(self, response=None, text=None, node=None, parent=None, expr=None): - if parent is not None: - self.doc = parent.doc - self.xmlNode = node - elif response: - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - elif text: - response = TextResponse(url='about:blank', \ - body=unicode_to_str(text, 'utf-8'), encoding='utf-8') - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - self.expr = expr - - def select(self, xpath): - if hasattr(self.xmlNode, 'xpathEval'): - self.doc.xpathContext.setContextNode(self.xmlNode) - xpath = unicode_to_str(xpath, 'utf-8') - try: - xpath_result = self.doc.xpathContext.xpathEval(xpath) - except libxml2.xpathError: - raise ValueError("Invalid XPath: %s" % xpath) - if hasattr(xpath_result, '__iter__'): - return SelectorList([self.__class__(node=node, parent=self, \ - expr=xpath) for node in xpath_result]) - else: - return SelectorList([self.__class__(node=xpath_result, \ - parent=self, expr=xpath)]) - else: - return SelectorList([]) - - def re(self, regex): - return extract_regex(regex, self.extract()) - - def extract(self): - if isinstance(self.xmlNode, basestring): - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - elif hasattr(self.xmlNode, 'serialize'): - if isinstance(self.xmlNode, libxml2.xmlDoc): - data = self.xmlNode.getRootElement().serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - elif isinstance(self.xmlNode, libxml2.xmlAttr): - # serialization doesn't work sometimes for xmlAttr types - text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') - else: - data = self.xmlNode.serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - else: - try: - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - except TypeError: # catched when self.xmlNode is a float - see tests - text = unicode(self.xmlNode) - return text - - def extract_unquoted(self): - """Get unescaped contents from the text node (no entities, no CDATA)""" - # TODO: this function should be deprecated. but what would be use instead? - if self.select('self::text()'): - return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') - else: - return u'' - - def register_namespace(self, prefix, uri): - self.doc.xpathContext.xpathRegisterNs(prefix, uri) - - def _get_libxml2_doc(self, response): - return xmlDoc_from_html(response) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) - - __repr__ = __str__ - - @deprecated(use_instead='XPathSelector.select') - def __call__(self, xpath): - return self.select(xpath) - - @deprecated(use_instead='XPathSelector.select') - def x(self, xpath): - return self.select(xpath) - - -class XmlXPathSelector(XPathSelector): - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_xml) - - -class HtmlXPathSelector(XPathSelector): - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_html) diff --git a/scrapy/tests/test_libxml2.py b/scrapy/tests/test_libxml2.py deleted file mode 100644 index e042de526..000000000 --- a/scrapy/tests/test_libxml2.py +++ /dev/null @@ -1,20 +0,0 @@ -from twisted.trial import unittest - -from scrapy.utils.test import libxml2debug -from scrapy import optional_features - - -class Libxml2Test(unittest.TestCase): - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_libxml2_bug_2_6_27(self): - # this test will fail in version 2.6.27 but passes on 2.6.29+ - import libxml2 - html = "123" - node = libxml2.htmlParseDoc(html, 'utf-8') - result = [str(r) for r in node.xpathEval('//text()')] - self.assertEquals(result, ['1', '2', '3']) - node.freeDoc() - diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 284c28db6..fc732a694 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -10,7 +10,7 @@ from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector -from scrapy.utils.test import libxml2debug + class XPathSelectorTestCase(unittest.TestCase): @@ -18,7 +18,6 @@ class XPathSelectorTestCase(unittest.TestCase): hxs_cls = HtmlXPathSelector xxs_cls = XmlXPathSelector - @libxml2debug def test_selector_simple(self): """Simple selector tests""" body = "

" @@ -43,23 +42,20 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) - @libxml2debug def test_selector_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8') xpath = self.hxs_cls(response) self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1']) - @libxml2debug def test_selector_same_type(self): """Test XPathSelector returning the same type in x() method""" text = '

test

' assert isinstance(self.xxs_cls(text=text).select("//p")[0], self.xxs_cls) - assert isinstance(self.hxs_cls(text=text).select("//p")[0], + assert isinstance(self.hxs_cls(text=text).select("//p")[0], self.hxs_cls) - @libxml2debug def test_selector_boolean_result(self): body = "

" response = TextResponse(url="http://example.com", body=body) @@ -67,7 +63,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1']) self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0']) - @libxml2debug def test_selector_xml_html(self): """Test that XML and HTML XPathSelector's behave differently""" @@ -80,7 +75,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(self.hxs_cls(text=text).select("//div").extract(), [u'

Hello

']) - @libxml2debug def test_selector_nested(self): """Nested selector tests""" body = """ @@ -109,13 +103,11 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(divtwo.select("./li").extract(), []) - @libxml2debug def test_dont_strip(self): hxs = self.hxs_cls(text='
fff: zzz
') self.assertEqual(hxs.select("//text()").extract(), [u'fff: ', u'zzz']) - @libxml2debug def test_selector_namespaces_simple(self): body = """ @@ -131,7 +123,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//somens:a/text()").extract(), [u'take this']) - @libxml2debug def test_selector_namespaces_multiple(self): body = """ Name: Mary
    @@ -177,14 +167,12 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), ["10", "20"]) - @libxml2debug def test_selector_re_intl(self): body = """
    Evento: cumplea\xc3\xb1os
    """ response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8') x = self.hxs_cls(response) self.assertEqual(x.select("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) - @libxml2debug def test_selector_over_text(self): hxs = self.hxs_cls(text='lala') self.assertEqual(hxs.extract(), @@ -199,7 +187,6 @@ class XPathSelectorTestCase(unittest.TestCase): [u'lala']) - @libxml2debug def test_selector_invalid_xpath(self): response = XmlResponse(url="http://example.com", body="") x = self.hxs_cls(response) @@ -213,7 +200,6 @@ class XPathSelectorTestCase(unittest.TestCase): else: raise AssertionError("A invalid XPath does not raise an exception") - @libxml2debug def test_http_header_encoding_precedence(self): # u'\xa3' = pound symbol in unicode # u'\xc2\xa3' = pound symbol in utf-8 @@ -233,14 +219,12 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x.select("//span[@id='blank']/text()").extract(), [u'\xa3']) - @libxml2debug def test_empty_bodies(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_null_bytes(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', \ @@ -249,7 +233,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence r1 = TextResponse('http://www.example.com', \ @@ -258,7 +241,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_select_on_unevaluable_nodes(self): r = self.hxs_cls(text=u'some text') # Text node @@ -270,7 +252,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x1.extract(), [u'big']) self.assertEquals(x1.select('.//text()').extract(), []) - @libxml2debug def test_select_on_text_nodes(self): r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") @@ -279,7 +260,6 @@ class XPathSelectorTestCase(unittest.TestCase): x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x1.extract(), [u'Options:']) - @libxml2debug def test_nested_select_on_text_nodes(self): # FIXME: does not work with lxml backend [upstream] r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') @@ -289,7 +269,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x2.extract(), [u'Options:']) test_nested_select_on_text_nodes.skip = True - @libxml2debug def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" for cls in [self.xs_cls, self.hxs_cls, self.xxs_cls]: diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py deleted file mode 100644 index 5dabd2d01..000000000 --- a/scrapy/tests/test_selector_libxml2.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Selectors tests, specific for libxml2 backend -""" - -from twisted.trial import unittest -from scrapy import optional_features - - -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector -from scrapy.selector.libxml2document import Libxml2Document -from scrapy.utils.test import libxml2debug -from scrapy.tests import test_selector - - -class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase): - - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_null_bytes(self): - hxs = HtmlXPathSelector(text='la\x00la') - self.assertEqual(hxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='la\x00la') - self.assertEqual(xxs.extract(), - u'lala') - - @libxml2debug - def test_unquote(self): - xmldoc = '\n'.join(( - '', - ' lala', - ' ', - ' blabla&moreatestoh', - ' PPPPppp&la]]>', - ' ', - ' pff', - '')) - xxs = XmlXPathSelector(text=xmldoc) - - self.assertEqual(xxs.extract_unquoted(), u'') - - self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) - self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n pff\n']) - - self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) - self.assertEqual(xxs.select('//text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n blabla&more', - u'a', - u'test', - u'oh\n ', - u'lalalal&pppppPPPPppp&la', - u'\n ', - u'\n pff\n']) - - -class Libxml2DocumentTest(unittest.TestCase): - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_response_libxml2_caching(self): - r1 = HtmlResponse('http://www.example.com', body='') - r2 = r1.copy() - - doc1 = Libxml2Document(r1) - doc2 = Libxml2Document(r1) - doc3 = Libxml2Document(r2) - - # make sure it's cached - assert doc1 is doc2 - assert doc1.xmlDoc is doc2.xmlDoc - assert doc1 is not doc3 - assert doc1.xmlDoc is not doc3.xmlDoc - - # don't leave libxml2 documents in memory to avoid wrong libxml2 leaks reports - del doc1, doc2, doc3 - - @libxml2debug - def test_null_char(self): - # make sure bodies with null char ('\x00') don't raise a TypeError exception - self.body_content = 'test problematic \x00 body' - response = TextResponse('http://example.com/catalog/product/blabla-123', - headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) - Libxml2Document(response) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 352861ae6..7ea674d62 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -39,6 +39,7 @@ class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): xxs.remove_namespaces() self.assertEqual(len(xxs.select("//link/@type")), 2) + class Libxml2DocumentTest(unittest.TestCase): def test_caching(self): @@ -53,9 +54,6 @@ class Libxml2DocumentTest(unittest.TestCase): assert doc1 is doc2 assert doc1 is not doc3 - # don't leave documents in memory to avoid wrong libxml2 leaks reports - del doc1, doc2, doc3 - def test_null_char(self): # make sure bodies with null char ('\x00') don't raise a TypeError exception self.body_content = 'test problematic \x00 body' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index a931443a6..2695a2921 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -6,30 +6,6 @@ import os from twisted.trial.unittest import SkipTest -def libxml2debug(testfunction): - """Decorator for debugging libxml2 memory leaks inside a function. - - We've found libxml2 memory leaks are something very weird, and can happen - sometimes depending on the order where tests are run. So this decorator - enables libxml2 memory leaks debugging only when the environment variable - LIBXML2_DEBUGLEAKS is set. - - """ - try: - import libxml2 - except ImportError: - return testfunction - def newfunc(*args, **kwargs): - libxml2.debugMemory(1) - testfunction(*args, **kwargs) - libxml2.cleanupParser() - leaked_bytes = libxml2.debugMemory(0) - assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes - - if 'LIBXML2_DEBUGLEAKS' in os.environ: - return newfunc - else: - return testfunction def assert_aws_environ(): """Asserts the current environment is suitable for running AWS testsi.