diff --git a/scrapy/selector/document.py b/scrapy/selector/document.py deleted file mode 100644 index ba75053ea..000000000 --- a/scrapy/selector/document.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -This module contains a simple class (Libxml2Document) which provides cache and -garbage collection to libxml2 documents (xmlDoc). -""" - -import weakref - -from scrapy.utils.trackref import object_ref -from .factories import xmlDoc_from_html - -class Libxml2Document(object_ref): - - cache = weakref.WeakKeyDictionary() - __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] - - def __new__(cls, response, factory=xmlDoc_from_html): - cache = cls.cache.setdefault(response, {}) - if factory not in cache: - obj = object_ref.__new__(cls) - obj.xmlDoc = factory(response) - obj.xpathContext = obj.xmlDoc.xpathNewContext() - cache[factory] = obj - return cache[factory] - - def __del__(self): - # we must call both cleanup functions, so we try/except all exceptions - # to make sure one doesn't prevent the other from being called - # this call sometimes raises a "NoneType is not callable" TypeError - # so the try/except block silences them - try: - self.xmlDoc.freeDoc() - except: - pass - try: - self.xpathContext.xpathFreeContext() - except: - pass - - def __str__(self): - return "" % self.xmlDoc.name diff --git a/scrapy/selector/factories.py b/scrapy/selector/factories.py deleted file mode 100644 index 44dc4f94f..000000000 --- a/scrapy/selector/factories.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module provides functions for generating libxml2 documents (xmlDoc). - -Constructors must receive a Response object and return a xmlDoc object. -""" - -import libxml2 - -xml_parser_options = libxml2.XML_PARSE_RECOVER + \ - libxml2.XML_PARSE_NOERROR + \ - libxml2.XML_PARSE_NOWARNING - -html_parser_options = libxml2.HTML_PARSE_RECOVER + \ - libxml2.HTML_PARSE_NOERROR + \ - libxml2.HTML_PARSE_NOWARNING - -utf8_encodings = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) - -def body_as_utf8(response): - if response.encoding in utf8_encodings: - return response.body - else: - return response.body_as_unicode().encode('utf-8') - -def xmlDoc_from_html(response): - """Return libxml2 doc for HTMLs""" - utf8body = body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ - html_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', html_parser_options) - return lxdoc - -def xmlDoc_from_xml(response): - """Return libxml2 doc for XMLs""" - utf8body = body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ - xml_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', xml_parser_options) - return lxdoc diff --git a/scrapy/selector/libxml2document.py b/scrapy/selector/libxml2document.py new file mode 100644 index 000000000..a1b903772 --- /dev/null +++ b/scrapy/selector/libxml2document.py @@ -0,0 +1,80 @@ +""" +This module contains a simple class (Libxml2Document) which provides cache and +garbage collection to libxml2 documents (xmlDoc). +""" + +import weakref +import libxml2 +from scrapy.utils.trackref import object_ref + +xml_parser_options = libxml2.XML_PARSE_RECOVER + \ + libxml2.XML_PARSE_NOERROR + \ + libxml2.XML_PARSE_NOWARNING + +html_parser_options = libxml2.HTML_PARSE_RECOVER + \ + libxml2.HTML_PARSE_NOERROR + \ + libxml2.HTML_PARSE_NOWARNING + + +_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) +def _body_as_utf8(response): + if response.encoding in _UTF8_ENCODINGS: + return response.body + else: + return response.body_as_unicode().encode('utf-8') + + +def xmlDoc_from_html(response): + """Return libxml2 doc for HTMLs""" + utf8body = _body_as_utf8(response) or ' ' + try: + lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ + html_parser_options) + except TypeError: # libxml2 doesn't parse text with null bytes + lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', html_parser_options) + return lxdoc + + +def xmlDoc_from_xml(response): + """Return libxml2 doc for XMLs""" + utf8body = _body_as_utf8(response) or ' ' + try: + lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ + xml_parser_options) + except TypeError: # libxml2 doesn't parse text with null bytes + lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', xml_parser_options) + return lxdoc + + +class Libxml2Document(object_ref): + + cache = weakref.WeakKeyDictionary() + __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] + + def __new__(cls, response, factory=xmlDoc_from_html): + cache = cls.cache.setdefault(response, {}) + if factory not in cache: + obj = object_ref.__new__(cls) + obj.xmlDoc = factory(response) + obj.xpathContext = obj.xmlDoc.xpathNewContext() + cache[factory] = obj + return cache[factory] + + def __del__(self): + # we must call both cleanup functions, so we try/except all exceptions + # to make sure one doesn't prevent the other from being called + # this call sometimes raises a "NoneType is not callable" TypeError + # so the try/except block silences them + try: + self.xmlDoc.freeDoc() + except: + pass + try: + self.xpathContext.xpathFreeContext() + except: + pass + + def __str__(self): + return "" % self.xmlDoc.name diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py index 9342c9639..7bd3b8a38 100644 --- a/scrapy/selector/libxml2sel.py +++ b/scrapy/selector/libxml2sel.py @@ -9,8 +9,7 @@ from scrapy.utils.python import unicode_to_str from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated -from .factories import xmlDoc_from_html, xmlDoc_from_xml -from .document import Libxml2Document +from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml from .list import XPathSelectorList __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ diff --git a/scrapy/selector/lxmldocument.py b/scrapy/selector/lxmldocument.py new file mode 100644 index 000000000..065ce222b --- /dev/null +++ b/scrapy/selector/lxmldocument.py @@ -0,0 +1,31 @@ +""" +This module contains a simple class (LxmlDocument) which provides cache and +garbage collection to lxml element tree documents. +""" + +import weakref +from lxml import etree +from scrapy.utils.trackref import object_ref + + +def _factory(response, parser_cls): + url = response.url + body = response.body_as_unicode().strip().encode('utf8') or '' + parser = parser_cls(recover=True, encoding='utf8') + return etree.fromstring(body, parser=parser, base_url=url) + + +class LxmlDocument(object_ref): + + cache = weakref.WeakKeyDictionary() + __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] + + def __new__(cls, response, parser=etree.HTMLParser): + cache = cls.cache.setdefault(response, {}) + if parser not in cache: + obj = object_ref.__new__(cls) + cache[parser] = _factory(response, parser) + return cache[parser] + + def __str__(self): + return "" % self.root.tag diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index bb0a4e1ec..af6839474 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -9,42 +9,35 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.python import unicode_to_str from scrapy.utils.decorator import deprecated from scrapy.http import TextResponse +from .lxmldocument import LxmlDocument from .list import XPathSelectorList + __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] + 'XPathSelectorList'] + class XPathSelector(object_ref): - __slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '_xpathev', \ - '__weakref__'] + __slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__'] _parser = etree.HTMLParser _tostring_method = 'html' - def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None): - if text: - self.response = TextResponse(url='about:blank', \ + def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None): + if text is not None: + response = TextResponse(url='about:blank', \ body=unicode_to_str(text, 'utf-8'), encoding='utf-8') - else: - self.response = response - self._root = root - self._xpathev = None - self.namespaces = namespaces - self.expr = expr + if response is not None: + _root = LxmlDocument(response, self._parser) - @property - def root(self): - if self._root is None: - url = self.response.url - body = self.response.body_as_unicode().strip().encode('utf8') or '' - parser = self._parser(recover=True, encoding='utf8') - self._root = etree.fromstring(body, parser=parser, base_url=url) - assert self._root is not None, 'BUG lxml selector with None root' - return self._root + self.namespaces = namespaces + self.response = response + self._root = _root + self._expr = _expr def select(self, xpath): try: - xpathev = self.root.xpath + xpathev = self._root.xpath except AttributeError: return XPathSelectorList([]) @@ -56,7 +49,7 @@ class XPathSelector(object_ref): if type(result) is not list: result = [result] - result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) + result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces) for x in result] return XPathSelectorList(result) @@ -65,15 +58,15 @@ class XPathSelector(object_ref): def extract(self): try: - return etree.tostring(self.root, method=self._tostring_method, \ + return etree.tostring(self._root, method=self._tostring_method, \ encoding=unicode) except (AttributeError, TypeError): - if self.root is True: + if self._root is True: return u'1' - elif self.root is False: + elif self._root is False: return u'0' else: - return unicode(self.root) + return unicode(self._root) def register_namespace(self, prefix, uri): if self.namespaces is None: @@ -85,7 +78,7 @@ class XPathSelector(object_ref): def __str__(self): data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) __repr__ = __str__ diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py index 917ddaf96..e447a6297 100644 --- a/scrapy/tests/test_selector_libxml2.py +++ b/scrapy/tests/test_selector_libxml2.py @@ -7,10 +7,11 @@ import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector -from scrapy.selector.document import Libxml2Document +from scrapy.selector.libxml2document import Libxml2Document from scrapy.utils.test import libxml2debug from scrapy.tests import test_selector + class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase): xs_cls = XPathSelector diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 8050a2758..c8eca9679 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -2,35 +2,40 @@ Selectors tests, specific for lxml backend """ -from scrapy.http import TextResponse, XmlResponse -has_lxml = True -try: - from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector -except ImportError: - has_lxml = False -from scrapy.utils.test import libxml2debug +import unittest from scrapy.tests import test_selector +from scrapy.http import TextResponse, HtmlResponse +from scrapy.selector.lxmldocument import LxmlDocument +from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector + class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): - if has_lxml: - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - else: - skip = "lxml not available" + xs_cls = XPathSelector + hxs_cls = HtmlXPathSelector + xxs_cls = XmlXPathSelector - # XXX: this test was disabled because lxml behaves inconsistently when - # handling null bytes between different 2.2.x versions, but it may be due - # to differences in libxml2 too. it's also unclear what should be the - # proper behaviour (pablo - 26 oct 2010) - #@libxml2debug - #def test_null_bytes(self): - # hxs = HtmlXPathSelector(text='la\x00la') - # self.assertEqual(hxs.extract(), - # u'la') - # - # xxs = XmlXPathSelector(text='la\x00la') - # self.assertEqual(xxs.extract(), - # u'la') + +class Libxml2DocumentTest(unittest.TestCase): + + def test_caching(self): + r1 = HtmlResponse('http://www.example.com', body='') + r2 = r1.copy() + + doc1 = LxmlDocument(r1) + doc2 = LxmlDocument(r1) + doc3 = LxmlDocument(r2) + + # make sure it's cached + assert doc1 is doc2 + assert doc1 is not doc3 + + # don't leave documents in memory to avoid wrong libxml2 leaks reports + del doc1, doc2, doc3 + + def test_null_char(self): + # make sure bodies with null char ('\x00') don't raise a TypeError exception + self.body_content = 'test problematic \x00 body' + response = TextResponse('http://example.com/catalog/product/blabla-123', + headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) + LxmlDocument(response)